diff --git a/src/main/claude/claude-agent-sdk-control-requests.ts b/src/main/claude/claude-agent-sdk-control-requests.ts index 37417569b07a..f165822c9379 100644 --- a/src/main/claude/claude-agent-sdk-control-requests.ts +++ b/src/main/claude/claude-agent-sdk-control-requests.ts @@ -16,6 +16,15 @@ export class ClaudeControlRequestError extends Error { export const CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS = 30_000 +/** The deadline fired before the CLI answered. The request may still land, so this proves + * nothing about it either way; it is neither the CLI refusing nor the transport closing. */ +export class ClaudeControlRequestTimeoutError extends Error { + constructor(readonly subtype: string) { + super(`claude ${subtype} request timed out`) + this.name = 'ClaudeControlRequestTimeoutError' + } +} + /** The SDK closes a query out from under an in-flight control request with this exact message. */ const QUERY_CLOSED_MESSAGE = 'Query closed before response received' @@ -46,33 +55,34 @@ export type ClaudeControlOptions = { timeoutMs?: number } /** * Run one native Query control method under Orca's deadline and error classification. * - * The SDK owns correlation but applies no deadline, so the timeout stays here — and its - * message is load-bearing: the init proof matches on `claude initialize request timed out`. + * The SDK owns correlation but applies no deadline, so the timeout stays here. `null` means + * none: the request then settles only on the CLI's answer or on the query closing under it. * A closed query is a transport failure, not the CLI rejecting the request, so only the * latter is re-thrown as a `ClaudeControlRequestError` a caller may surface as a rejection. */ export function runClaudeControl( subtype: string, run: () => Promise, - timeoutMs: number = CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS + timeoutMs: number | null = CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS ): Promise { let timer: ReturnType | null = null + const request = Promise.resolve() + .then(run) + .catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error) + if (error instanceof ClaudeControlRequestError || message === QUERY_CLOSED_MESSAGE) { + throw error + } + throw new ClaudeControlRequestError(subtype, message) + }) + if (timeoutMs === null) { + return request + } const deadline = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new Error(`claude ${subtype} request timed out`)), timeoutMs) + timer = setTimeout(() => reject(new ClaudeControlRequestTimeoutError(subtype)), timeoutMs) timer.unref?.() }) - return Promise.race([ - Promise.resolve() - .then(run) - .catch((error: unknown) => { - const message = error instanceof Error ? error.message : String(error) - if (error instanceof ClaudeControlRequestError || message === QUERY_CLOSED_MESSAGE) { - throw error - } - throw new ClaudeControlRequestError(subtype, message) - }), - deadline - ]).finally(() => { + return Promise.race([request, deadline]).finally(() => { if (timer) { clearTimeout(timer) } @@ -93,7 +103,8 @@ export type ClaudeControlSurface = { ) => Promise stopTask: (taskId: string, options?: ClaudeControlOptions) => Promise supportedModels: (options?: ClaudeControlOptions) => Promise - initializationResult: (options?: ClaudeControlOptions) => Promise + /** Untimed: a slow start is still a start, and the child's exit closes the query under it. */ + initializationResult: () => Promise getSettings: (options?: ClaudeControlOptions) => Promise /** The `/context` breakdown; older CLIs reject the request and the caller shows nothing. */ getContextUsage: (options?: ClaudeControlOptions) => Promise @@ -144,8 +155,8 @@ export function createClaudeControlSurface(query: Query): ClaudeControlSurface { ), supportedModels: (options) => runClaudeControl('list_models', () => query.supportedModels(), options?.timeoutMs), - initializationResult: (options) => - runClaudeControl('initialize', () => query.initializationResult(), options?.timeoutMs), + initializationResult: () => + runClaudeControl('initialize', () => query.initializationResult(), null), getContextUsage: (options) => runClaudeControl('get_context_usage', () => query.getContextUsage(), options?.timeoutMs), getSettings: (options) => { diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index 23e06c76436a..51ffa085163d 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -515,10 +515,12 @@ describe('Claude stream-json connection', () => { } }) const connection = await open(launchFor(scenario)) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies every session member the option paths under test read. const session = { connection, options: new Map(), - reportedOptions: {} + reportedOptions: {}, + startup: { state: 'proven' } } as unknown as ClaudeSession const options = await readClaudeStructuredSessionOptions(session, 5_000) @@ -567,17 +569,15 @@ describe('Claude stream-json connection', () => { expect(JSON.stringify(diagnostic)).not.toContain('secret') }) - it('reports an unauthenticated start through the init deadline instead of hanging', async () => { - // The scripted CLI never answers, which is the shape of a silently unauthenticated CLI. - const scenario = scriptScenario([HOLD_OPEN]) + it('settles an unanswered initialize when the child exits, with no timer of its own', async () => { + // The scripted CLI never answers, then leaves: only its exit can settle the request. + const scenario = scriptScenario([{ stderr: 'claude: not signed in\n' }, { exit: 1 }]) const connection = await open({ ...launchFor(scenario), env: { ...launchFor(scenario).env, ORCA_SDK_CONTRACT_IGNORE_CONTROL_REQUESTS: '1' } }) - await expect(connection.initializationResult({ timeoutMs: 200 })).rejects.toThrow( - 'claude initialize request timed out' - ) + await expect(connection.initializationResult()).rejects.toThrow() }) it('reports a self-exit with its status, stderr, and observed tree verdict', async () => { diff --git a/src/main/claude/claude-structured-dispatch-test-support.ts b/src/main/claude/claude-structured-dispatch-test-support.ts index 605c90728057..c9e16084a7da 100644 --- a/src/main/claude/claude-structured-dispatch-test-support.ts +++ b/src/main/claude/claude-structured-dispatch-test-support.ts @@ -4,6 +4,7 @@ import { retireClaudeDispatchWaiters } from './claude-structured-dispatch' import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' +import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): ClaudeSession { return { @@ -28,7 +29,8 @@ export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): C restoreSkippedOptions: new Set(), capabilities: [], events: undefined, - translator: null + translator: null, + startup: { ...createClaudeSessionStartupGate(), state: 'proven' } } } diff --git a/src/main/claude/claude-structured-dispatch.ts b/src/main/claude/claude-structured-dispatch.ts index 73c14d155f4d..d93a8ecab0a3 100644 --- a/src/main/claude/claude-structured-dispatch.ts +++ b/src/main/claude/claude-structured-dispatch.ts @@ -33,6 +33,12 @@ import { claudeUserMessageWasProvablyUnwritten } from './claude-agent-sdk-user-message-queue' import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' +import { + claudeStartupFailureReason, + claudeStartupHoldsWrites, + failClaudeStartupGate, + holdClaudeStartupWrite +} from './claude-structured-session-startup-gate' const MAX_ACTIVE_DISPATCH_WAITERS = 64 @@ -220,6 +226,7 @@ export function settleCancelledClaudeDispatchWaiters( * Retired rather than dropped: their identities stay joinable, bounded by * `MAX_RETIRED_DISPATCH_WAITERS`. */ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { + failClaudeStartupGate(session, new Error('claude stream-json ended before startup completed')) for (const waiter of session.dispatchWaiters.splice(0)) { retireWaiter(session, waiter) waiter.resolve(null) @@ -229,7 +236,8 @@ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { export async function dispatchClaudeTurn( session: ClaudeSession, input: { clientMessageId?: string; body: AgentJournalMessageItem; requestedAt?: number }, - beforeDispatch?: () => Promise + beforeDispatch?: () => Promise, + onSettledLate?: ClaudeLateDispatchSettlement ): Promise { let content: unknown[] try { @@ -240,6 +248,10 @@ export async function dispatchClaudeTurn( if (session.dispatchWaiters.length >= MAX_ACTIVE_DISPATCH_WAITERS) { return { state: 'rejected', reason: DISPATCH_REJECTED_QUEUE_FULL } } + const startupFailure = claudeStartupFailureReason(session) + if (startupFailure) { + return { state: 'rejected', reason: startupFailure } + } // Read the sent content, not the journal blocks: only the mapped trailing prompt decides // whether Claude runs a command, so the two cannot disagree about which frame settles this. const acceptsResult = claudeDispatchInvokesSlashCommand(content) @@ -257,6 +269,21 @@ export async function dispatchClaudeTurn( input.requestedAt ?? null ) } + const message = { + type: 'user', + uuid: sentUuid, + message: { role: 'user', content }, + parent_tool_use_id: null, + session_id: session.providerSessionId + } + if (claudeStartupHoldsWrites(session)) { + return holdClaudeStartupWrite(session, { + message, + arm, + ...(beforeDispatch ? { beforeDispatch } : {}), + ...(onSettledLate ? { settleLate: onSettledLate } : {}) + }) + } const pending = { replay: beforeDispatch ? undefined : arm() } const authorize = beforeDispatch ? async () => { @@ -268,13 +295,6 @@ export async function dispatchClaudeTurn( } : undefined try { - const message = { - type: 'user', - uuid: sentUuid, - message: { role: 'user', content }, - parent_tool_use_id: null, - session_id: session.providerSessionId - } await (authorize ? session.connection.send(message, authorize) : session.connection.send(message)) diff --git a/src/main/claude/claude-structured-effort-reporting.test.ts b/src/main/claude/claude-structured-effort-reporting.test.ts index be022d869568..7cf59ee555bc 100644 --- a/src/main/claude/claude-structured-effort-reporting.test.ts +++ b/src/main/claude/claude-structured-effort-reporting.test.ts @@ -22,6 +22,7 @@ function sessionWith( listed?: { model: string; catalog: readonly Record[] } ) { return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies every session member the option paths under test read. session: { options: new Map(listed ? [['model', listed.model]] : []), reportedOptions: {} as { model?: string; effort?: string }, @@ -47,7 +48,8 @@ function sessionWith( ? { applied: {}, effective: {}, sources: {} } : { applied: { effort: reported }, effective: { effortLevel: reported }, sources: {} } } - } + }, + startup: { state: 'proven' } } as unknown as ClaudeSession, calls } diff --git a/src/main/claude/claude-structured-init-deadline.ts b/src/main/claude/claude-structured-init-deadline.ts deleted file mode 100644 index f3acd6c3af98..000000000000 --- a/src/main/claude/claude-structured-init-deadline.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { ClaudeInitObservation } from './claude-structured-init-proof' -import { claudeInitializationAuthError } from './claude-structured-init-proof' -import type { ClaudeStreamJsonConnection } from './claude-stream-json-connection' -import { AgentSessionAcquisitionRefusal } from '../native-chat/agent-session-wire/structured-agent-session-adapter' - -export type ClaudeInitDeadline = { - promise: Promise - resolve: (init: ClaudeInitObservation) => void - reject: (error: Error) => void - start: () => void - clear: () => void -} - -export function claudeInitTimeoutError( - sessionId: string, - timeoutMs: number -): AgentSessionAcquisitionRefusal { - return new AgentSessionAcquisitionRefusal( - `Claude did not finish starting session ${sessionId} within ${Math.ceil(timeoutMs / 1000)} seconds. Verify the selected Claude account is signed in and CLAUDE_CONFIG_DIR contains valid credentials, then retry; no SessionStart or system/init proof arrived.` - ) -} - -export async function requestClaudeInitialization( - connection: ClaudeStreamJsonConnection, - sessionId: string, - timeoutMs: number -): Promise { - try { - const result = await connection.initializationResult({ timeoutMs }) - const authError = claudeInitializationAuthError(result) - if (authError) { - throw authError - } - return result - } catch (error) { - if (error instanceof Error && error.message === 'claude initialize request timed out') { - throw claudeInitTimeoutError(sessionId, timeoutMs) - } - throw error - } -} - -export function createClaudeInitDeadline(sessionId: string, timeoutMs: number): ClaudeInitDeadline { - let resolve = (_init: ClaudeInitObservation): void => {} - let reject = (_error: Error): void => {} - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise - reject = rejectPromise - }) - void promise.catch(() => {}) - let timer: ReturnType | null = null - - return { - promise, - resolve, - reject, - start: () => { - timer = setTimeout(() => reject(claudeInitTimeoutError(sessionId, timeoutMs)), timeoutMs) - timer.unref?.() - }, - clear: () => { - if (timer) { - clearTimeout(timer) - timer = null - } - } - } -} diff --git a/src/main/claude/claude-structured-launch-resolution.test.ts b/src/main/claude/claude-structured-launch-resolution.test.ts index ac5a68445dab..c54d1a5179f3 100644 --- a/src/main/claude/claude-structured-launch-resolution.test.ts +++ b/src/main/claude/claude-structured-launch-resolution.test.ts @@ -1,7 +1,7 @@ import { chmodSync, mkdtempSync, mkdirSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { delimiter, join } from 'node:path' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import type { AgentSessionRecord } from '../../shared/agent-session-record' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { AgentSessionRecordStore } from '../runtime/agent-session-record-store' @@ -58,7 +58,8 @@ function resolverFor( resolveEnv?: () => Record, stripAuthEnv = false, // Manual by default so a test that is not about permissions is not silently about them. - agentDefaultArgs: Record = { claude: '' } + agentDefaultArgs: Record = { claude: '' }, + hasTranscript: () => Promise = async () => true ) { return createClaudeStructuredLaunchResolver({ store: { getRecord: () => value } as unknown as AgentSessionRecordStore, @@ -66,6 +67,7 @@ function resolverFor( resolveCommand: () => '/usr/local/bin/claude', resolveAuthPolicy: () => ({ stripAuthEnv }), resolvePermissionMode: () => claudeStructuredPermissionModeForSettings({ agentDefaultArgs }), + hasTranscript, ...(resolveEnv ? { resolveEnv } : {}) }) } @@ -115,7 +117,8 @@ describe('claude structured launch resolution', () => { cwd: '/repos/workspace-1', claudeConfigDir: '/home/work/.claude', resumeLeafUuid: null, - resumed: false + resumesTranscript: false, + continuesChain: false }) expect(first.options).toEqual({ includePartialMessages: true, @@ -149,7 +152,8 @@ describe('claude structured launch resolution', () => { expect(launch).toMatchObject({ providerSessionId: 'provider-current', resumeLeafUuid: 'leaf-current', - resumed: true + resumesTranscript: true, + continuesChain: true }) expect(launch.options.resume).toBe('provider-current') // Claude owns where the conversation continues; a stored leaf would cut or branch it. @@ -202,6 +206,37 @@ describe('claude structured launch resolution', () => { expect(launch.options).not.toHaveProperty('resumeSessionAt') }) + it('launches a leafless head fresh under its own id when Claude never wrote its transcript', async () => { + // A start that failed before its first turn: `--resume` would exit "No conversation found". + const hasTranscript = vi.fn(async () => false) + const launch = await resolverFor( + record({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the resolver reads only each link's handle. + providerHandleChain: [ + { handle: { provider: 'claude', sessionId: 'provider-current', leafUuid: null } } + ] as AgentSessionRecord['providerHandleChain'] + }), + undefined, + false, + { claude: '' }, + hasTranscript + )({ identity: identityAt(null) }) + + expect(hasTranscript).toHaveBeenCalledWith({ + providerSessionId: 'provider-current', + claudeConfigDir: expect.any(String) + }) + expect(launch.options.resume).toBeUndefined() + expect(launch.options.sessionId).toBe('provider-current') + expect(launch).toMatchObject({ + providerSessionId: 'provider-current', + resumeLeafUuid: null, + resumesTranscript: false, + // Launching the id fresh does not start a new conversation: the child continues the chain. + continuesChain: true + }) + }) + // Agent Permissions is stored as the bypass flag inside the launch arguments, so presence of // that flag — not the whole string — is what Yolo means, exactly as a terminal launch reads it. it.each([ diff --git a/src/main/claude/claude-structured-launch-resolution.ts b/src/main/claude/claude-structured-launch-resolution.ts index 565160117756..151d017ebcc7 100644 --- a/src/main/claude/claude-structured-launch-resolution.ts +++ b/src/main/claude/claude-structured-launch-resolution.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto' +import { join } from 'node:path' import type { Options as ClaudeAgentSdkOptions, PermissionMode @@ -25,6 +26,7 @@ import { type ClaudeManagedAccountGateSettings } from '../native-chat/claude-structured-managed-account-support' import { resolveClaudeCommand } from '../codex-cli/command' +import { resolveSessionFilePath } from '../native-chat/session-file-resolver' import { withoutInheritedClaudeConfigDir } from './claude-config-dir-pin' import type { AgentSessionRecordStore } from '../runtime/agent-session-record-store' @@ -95,7 +97,11 @@ export type ClaudeStructuredLaunch = { providerSessionId: string /** The previous head leaf, carried into the publication link; never a resume argument. */ resumeLeafUuid: string | null - resumed: boolean + /** Launch mode: `--resume` of a transcript Claude wrote, rather than starting the id fresh. */ + resumesTranscript: boolean + /** Lineage: the record's chain already heads this provider session, so the child continues it + * even when no transcript exists to `--resume`. Never derived from the launch mode. */ + continuesChain: boolean } export type ClaudeStructuredLaunchResolverDeps = { @@ -121,6 +127,21 @@ export type ClaudeStructuredLaunchResolverDeps = { authSwitchSettleTimeoutMs?: number /** Account state for the managed-account gate; null when it cannot be read, which refuses. */ readManagedAccountGate?: () => ClaudeManagedAccountGateSettings | null + /** Whether Claude wrote a transcript for this id; defaults to the transcript resolver. */ + hasTranscript?: (input: { + providerSessionId: string + claudeConfigDir: string + }) => Promise +} + +async function claudeTranscriptExists(input: { + providerSessionId: string + claudeConfigDir: string +}): Promise { + const path = await resolveSessionFilePath('claude', input.providerSessionId, { + claudeProjectsDir: join(input.claudeConfigDir, 'projects') + }) + return path !== null } /** @@ -193,6 +214,16 @@ export function createClaudeStructuredLaunchResolver( head?.handle.provider === 'claude' ? head.handle.sessionId : claudeSessionIdForOrcaSession(identity.sessionId) + const continuesChain = head?.handle.provider === 'claude' + // A start that failed before its first turn wrote no transcript, and `--resume` of an absent + // one exits; launch that id fresh instead. With a transcript, `--session-id` would collide. + const resumesTranscript = + head?.handle.provider === 'claude' && + (head.handle.leafUuid !== null || + (await (deps.hasTranscript ?? claudeTranscriptExists)({ + providerSessionId, + claudeConfigDir: record.accountHome.path + }))) // `record.launchArgs` is deliberately not read: the configured CLI arguments are a terminal // concern, and the permission mode they used to smuggle in is an owned provider option now. const permission = claudeStructuredPermissionOptions( @@ -243,16 +274,16 @@ export function createClaudeStructuredLaunchResolver( ...permission, extraArgs: { ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, ...permission.extraArgs }, // Claude owns where a resumed conversation continues; the stored leaf is Orca's bookkeeping. - ...(head?.handle.provider === 'claude' - ? { resume: providerSessionId } - : { sessionId: providerSessionId }) + ...(resumesTranscript ? { resume: providerSessionId } : { sessionId: providerSessionId }) }, cwd: await deps.resolveWorkspacePath(record.location.workspaceId), env, claudeConfigDir: record.accountHome.path, providerSessionId, - resumeLeafUuid: head?.handle.provider === 'claude' ? head.handle.leafUuid : null, - resumed: head?.handle.provider === 'claude' + resumeLeafUuid: + resumesTranscript && head?.handle.provider === 'claude' ? head.handle.leafUuid : null, + resumesTranscript, + continuesChain } } } diff --git a/src/main/claude/claude-structured-model-confirmation.test.ts b/src/main/claude/claude-structured-model-confirmation.test.ts index 7bd8f6291194..96d7e9daa391 100644 --- a/src/main/claude/claude-structured-model-confirmation.test.ts +++ b/src/main/claude/claude-structured-model-confirmation.test.ts @@ -168,6 +168,7 @@ describe('Claude effort the settings readback cannot report', () => { calls: string[] = [] ): { session: ClaudeSession; calls: string[] } { return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies every session member the option paths under test read. session: { options: new Map([['model', 'sonnet']]), reportedOptions: {}, @@ -189,7 +190,8 @@ describe('Claude effort the settings readback cannot report', () => { sources: {} } } - } + }, + startup: { state: 'proven' } } as unknown as ClaudeSession, calls } diff --git a/src/main/claude/claude-structured-model-preflight.test.ts b/src/main/claude/claude-structured-model-preflight.test.ts index e4f5d00b1964..930947e112db 100644 --- a/src/main/claude/claude-structured-model-preflight.test.ts +++ b/src/main/claude/claude-structured-model-preflight.test.ts @@ -18,6 +18,7 @@ const HAIKU = { function sessionWith(catalog: readonly Record[] | 'unavailable') { const calls: string[] = [] return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies every session member the option paths under test read. session: { options: new Map(), reportedOptions: {} as { model?: string; effort?: string }, @@ -36,7 +37,8 @@ function sessionWith(catalog: readonly Record[] | 'unavailable' setModel: async (model: string) => { calls.push(`set_model:${model}`) } - } + }, + startup: { state: 'proven' } } as unknown as ClaudeSession, calls } diff --git a/src/main/claude/claude-structured-option-confirmation.test.ts b/src/main/claude/claude-structured-option-confirmation.test.ts index ca7b8b70f1c8..d5ac5fd4d2dd 100644 --- a/src/main/claude/claude-structured-option-confirmation.test.ts +++ b/src/main/claude/claude-structured-option-confirmation.test.ts @@ -162,6 +162,7 @@ describe('confirmation never outlives the write it belongs to', () => { it('drops an earlier effort confirmation when the value changes', async () => { const calls: string[] = [] let reported = 'low' + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies every session member the option paths under test read. const session = { options: new Map([['model', 'sonnet']]), reportedOptions: {}, @@ -177,7 +178,8 @@ describe('confirmation never outlives the write it belongs to', () => { effective: { effortLevel: reported }, sources: {} }) - } + }, + startup: { state: 'proven' } } as unknown as ClaudeSession await setClaudeStructuredOption(session, { key: 'effort', value: 'low' }, undefined) diff --git a/src/main/claude/claude-structured-options.test.ts b/src/main/claude/claude-structured-options.test.ts index dff0c7f035b6..ec5e5477f45a 100644 --- a/src/main/claude/claude-structured-options.test.ts +++ b/src/main/claude/claude-structured-options.test.ts @@ -4,9 +4,15 @@ import { setClaudeStructuredOption } from './claude-structured-options' import type { ClaudeSession } from './claude-structured-session-state' +import { + ClaudeControlRequestError, + ClaudeControlRequestTimeoutError +} from './claude-agent-sdk-control-requests' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' +import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' import { + claudeStructuredSessionOptionsFrom, observeClaudeFastModeFacts, readClaudeStructuredSessionOptions } from './claude-structured-session-options' @@ -39,7 +45,8 @@ function sessionFor(setModel: ClaudeSession['connection']['setModel']): ClaudeSe restoreSkippedOptions: new Set(), capabilities: [], events: undefined, - translator: null + translator: null, + startup: { ...createClaudeSessionStartupGate(), state: 'proven' } } } @@ -438,3 +445,45 @@ describe('Claude Fast mode reported by the session frame alone', () => { expect(result.current.fastMode).toBeUndefined() }) }) + +describe('Claude structured option restore under the request deadline', () => { + it('keeps a saved choice the CLI never answered as wanted but unconfirmed, and drops a refused one', async () => { + const session = sessionFor(async () => { + throw new ClaudeControlRequestTimeoutError('set_model') + }) + session.connection.applyFlagSettings = async () => { + throw new ClaudeControlRequestTimeoutError('apply_flag_settings') + } + session.connection.setPermissionMode = async () => { + throw new ClaudeControlRequestError('set_permission_mode', 'unknown mode') + } + // Startup already read the CLI's own model and effort, and vouched for them. + session.reportedOptions = { model: 'claude-sonnet-5', effort: 'medium' } + session.confirmedOptions.add('effort') + session.options = new Map([ + ['model', 'sonnet'], + ['effort', 'high'], + ['permissionMode', 'plan'] + ]) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await expect(restoreClaudeStructuredSessionOptions(session, 10)).resolves.toBeUndefined() + + expect(Object.fromEntries(session.options)).toEqual({ model: 'sonnet', effort: 'high' }) + expect([...session.restoreSkippedOptions]).toEqual(['permissionMode']) + expect(claudeStructuredSessionOptionsFrom(session, null).current).toEqual({ + model: 'sonnet', + effort: 'high' + }) + }) + + it("keeps a timed-out client write as the deadline's own error, not a rejection", async () => { + const session = sessionFor(async () => { + throw new ClaudeControlRequestTimeoutError('set_model') + }) + + await expect( + setClaudeStructuredOption(session, { key: 'model', value: 'sonnet' }, 10) + ).rejects.toBeInstanceOf(ClaudeControlRequestTimeoutError) + }) +}) diff --git a/src/main/claude/claude-structured-options.ts b/src/main/claude/claude-structured-options.ts index 64417f9a16c7..4a879e4fb124 100644 --- a/src/main/claude/claude-structured-options.ts +++ b/src/main/claude/claude-structured-options.ts @@ -1,5 +1,6 @@ import type { EffortLevel, PermissionMode } from '@anthropic-ai/claude-agent-sdk' import { ClaudeControlRequestError } from './claude-stream-json-connection' +import { ClaudeControlRequestTimeoutError } from './claude-agent-sdk-control-requests' import { AgentSessionOptionRejectedError, isAgentSessionOptionRejectedError @@ -40,6 +41,23 @@ export function restoredClaudeStructuredSessionOptions( ) } +/** A client's write; the startup restore writes through `setClaudeStructuredOption` directly. */ +export function setClaudeStructuredSessionOption( + session: ClaudeSession, + input: { key: string; value: string }, + timeoutMs: number | undefined +): Promise>> { + // Each write is a control request the CLI answers only after initialize. + if (session.startup.state !== 'proven') { + return Promise.reject( + new AgentSessionOptionRejectedError( + 'Claude is still starting; options can be changed once it is ready.' + ) + ) + } + return setClaudeStructuredOption(session, input, timeoutMs) +} + export async function setClaudeStructuredOption( session: ClaudeSession, input: { key: string; value: string }, @@ -237,6 +255,16 @@ export async function restoreClaudeStructuredSessionOptions( try { await setClaudeStructuredOption(session, { key, value }, timeoutMs, value) } catch (error) { + // A write the CLI never answered must not fault a start that is otherwise fine. Silence is + // not a refusal, so the choice stays wanted, unconfirmed, and the next start retries it. + if (error instanceof ClaudeControlRequestTimeoutError) { + console.warn( + `[claude-structured] restore of ${key} for ${session.providerSessionId} was not answered in time; keeping it unconfirmed` + ) + session.options.set(key, value) + session.confirmedOptions.delete(key) + continue + } if (!isAgentSessionOptionRejectedError(error)) { throw error } diff --git a/src/main/claude/claude-structured-prompt-ownership.ts b/src/main/claude/claude-structured-prompt-ownership.ts index 399dd98506ab..e4c9de982d07 100644 --- a/src/main/claude/claude-structured-prompt-ownership.ts +++ b/src/main/claude/claude-structured-prompt-ownership.ts @@ -11,8 +11,13 @@ import { } from './claude-structured-control-actions' import type { ClaudeLateDispatchSettlement } from './claude-structured-dispatch' import type { ClaudeSession } from './claude-structured-session-state' +import { + claudeStartupHoldsWrites, + rejectClaudeStartupWrites +} from './claude-structured-session-startup-gate' +import { DISPATCH_REJECTED_CANCELLED } from '../../shared/structured-agent-session-dispatch-rejection' -/** Conservative user-facing window: below the 10s init and 30s control deadlines, trading +/** Conservative user-facing window: below the 30s control deadline, trading * residual slow-pump risk for ensuring delivery bookkeeping cannot block Stop indefinitely. */ export const CLAUDE_DISPATCH_ADMISSION_TIMEOUT_MS = 3_000 const CLAUDE_DISPATCH_ADMISSION_POLL_MS = 50 @@ -98,6 +103,15 @@ export async function cancelClaudeStructuredTurn(input: { const session = requireSession(sessions, request.sessionId) const acquisitionGeneration = session.acquisitionGeneration const prompt = request.prompt + // A held prompt was never written, so Stop withdraws it; the drain only writes what it still + // holds. Before startup lands nothing was written, so there is nothing to interrupt either. + let withdrewHeld = false + if (!prompt && claudeStartupHoldsWrites(session) && session.fence === request.fence) { + withdrewHeld = rejectClaudeStartupWrites(session, DISPATCH_REJECTED_CANCELLED) + if (session.startup.state === 'pending') { + return { cancelled: withdrewHeld } + } + } if (prompt && session.fence !== request.fence) { return { cancelled: false } } @@ -179,7 +193,7 @@ export async function cancelClaudeStructuredTurn(input: { } else if (claim) { session.prompts.releaseClaim(claim) } - return result + return withdrewHeld ? { ...result, cancelled: true } : result } catch (error) { if (claim && !interruptConfirmed) { session.prompts.releaseClaim(claim) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index cb3bb2b72ead..4181dab43c7a 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -51,7 +51,7 @@ function realAdapter( events: ClaudeStructuredSessionEvent[] = [], cwd = process.cwd() ): ClaudeStructuredSessionAdapter { - return new ClaudeStructuredSessionAdapter({ + const adapter = new ClaudeStructuredSessionAdapter({ resolveLaunch: async () => ({ pathToClaudeCodeExecutable: command, options: { ...CLAUDE_STRUCTURED_BASE_OPTIONS, sessionId: providerSessionId }, @@ -59,13 +59,21 @@ function realAdapter( claudeConfigDir, providerSessionId, resumeLeafUuid: null, - resumed: false + resumesTranscript: false, + continuesChain: false }), onEvent: (event) => events.push(event), readProcessStartTime: async () => 1, - now: () => 2, - initTimeoutMs: 5_000 + now: () => 2 }) + // These proofs read startup facts, which land after the session is published. + const acquire = adapter.acquire + adapter.acquire = async (input) => { + const acquisition = await acquire(input) + await adapter.drainStartup(input.identity.sessionId) + return acquisition + } + return adapter } function identity(providerSessionId: string): AgentSessionJournalIdentity { @@ -297,16 +305,20 @@ describe.skipIf(!realClaudeAvailable)('Claude structured real CLI handshake', () it('turns a real silent unauthenticated startup into sign-in guidance', async () => { const claudeConfigDir = await mkdtemp(join(tmpdir(), 'orca-claude-no-auth-')) const providerSessionId = randomUUID() - const adapter = realAdapter(providerSessionId, claudeConfigDir) + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) try { - await expect( - adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-no-auth' - }) - ).rejects.toThrow(/not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s) + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-no-auth' + }) + await adapter.drainObservedExits() + expect(events.find((event) => event.type === 'ended')).toMatchObject({ + reason: expect.stringMatching(/not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s), + startupUnproven: true + }) } finally { await adapter.closeAll() await rm(claudeConfigDir, { recursive: true, force: true }) diff --git a/src/main/claude/claude-structured-resume-point.test.ts b/src/main/claude/claude-structured-resume-point.test.ts index a11ffc154503..862909a2cd66 100644 --- a/src/main/claude/claude-structured-resume-point.test.ts +++ b/src/main/claude/claude-structured-resume-point.test.ts @@ -14,7 +14,12 @@ async function ownerMidSecondTurn(persisted: unknown[]) { const claude = fakeClaude() const adapter = adapterFor( claude, - { resumed: true, resumeLeafUuid: 'a3', options: { resume: PROVIDER_SESSION_ID } }, + { + resumesTranscript: true, + continuesChain: true, + resumeLeafUuid: 'a3', + options: { resume: PROVIDER_SESSION_ID } + }, [], persisted ) @@ -68,7 +73,12 @@ describe('Claude resume point is the last completed turn on every exit path', () const claude = fakeClaude() const adapter = adapterFor( claude, - { resumed: true, resumeLeafUuid: 'a3', options: { resume: PROVIDER_SESSION_ID } }, + { + resumesTranscript: true, + continuesChain: true, + resumeLeafUuid: 'a3', + options: { resume: PROVIDER_SESSION_ID } + }, [], persisted ) @@ -101,7 +111,12 @@ describe('Claude resume point is the last completed turn on every exit path', () const claude = fakeClaude() const adapter = adapterFor( claude, - { resumed: true, resumeLeafUuid: 'a3', options: { resume: PROVIDER_SESSION_ID } }, + { + resumesTranscript: true, + continuesChain: true, + resumeLeafUuid: 'a3', + options: { resume: PROVIDER_SESSION_ID } + }, events, [], undefined, diff --git a/src/main/claude/claude-structured-session-acquisition-options.ts b/src/main/claude/claude-structured-session-acquisition-options.ts index be50d04b2e2d..7557419dd08b 100644 --- a/src/main/claude/claude-structured-session-acquisition-options.ts +++ b/src/main/claude/claude-structured-session-acquisition-options.ts @@ -17,13 +17,18 @@ export function prepareClaudeStructuredSessionAcquisitionOptions(args: { settings: unknown initialization: unknown inputOptions: Readonly> | undefined - resumed: boolean + /** A fresh CLI session has not carried a per-session Fast opt-in over from anywhere. */ + resumesTranscript: boolean }) { const fastMode = readClaudeSettingsFastMode(args.settings) const fastModePerSessionOptIn = readClaudeSettingsFastModePerSessionOptIn(args.settings) const fastModeFacts = readClaudeFastModeFacts(args.initialization) const options = restoredClaudeStructuredSessionOptions(args.inputOptions) - if (!args.resumed && fastModePerSessionOptIn === true && options.get('fastMode') === 'true') { + if ( + !args.resumesTranscript && + fastModePerSessionOptIn === true && + options.get('fastMode') === 'true' + ) { options.delete('fastMode') } return { fastMode, fastModePerSessionOptIn, fastModeFacts, options } diff --git a/src/main/claude/claude-structured-session-acquisition-processless.test.ts b/src/main/claude/claude-structured-session-acquisition-processless.test.ts index 6a5e561b427a..85de77bbbd89 100644 --- a/src/main/claude/claude-structured-session-acquisition-processless.test.ts +++ b/src/main/claude/claude-structured-session-acquisition-processless.test.ts @@ -24,12 +24,13 @@ describe('Claude structured processless acquisition', () => { _launch, handlers = {} ) => { + // A failed spawn reports its error on a later tick, as child_process does. + setTimeout(() => handlers.onFault?.(fault), 0) const connection: ClaudeStreamJsonConnection = { pid: undefined, closed: true, exitVerdict: { root: 'processless', tree: 'exited' }, initializationResult: async () => { - handlers.onFault?.(fault) throw fault }, getSettings: async () => ({}), @@ -54,7 +55,8 @@ describe('Claude structured processless acquisition', () => { claudeConfigDir: '/accounts/claude', providerSessionId: PROVIDER_SESSION_ID, resumeLeafUuid: null, - resumed: false + resumesTranscript: false, + continuesChain: false }), openConnection }) diff --git a/src/main/claude/claude-structured-session-acquisition.ts b/src/main/claude/claude-structured-session-acquisition.ts index 74189edc37a2..d0d739abde43 100644 --- a/src/main/claude/claude-structured-session-acquisition.ts +++ b/src/main/claude/claude-structured-session-acquisition.ts @@ -8,31 +8,18 @@ import { isClaudeAuthSwitchInProgress } from '../claude-accounts/live-pty-gate' import { openClaudeStreamJsonConnection } from './claude-stream-json-connection' import { buildClaudePermissionCallbacks } from './claude-structured-inbound-control' import { resolveClaudeReplayTurn } from './claude-structured-dispatch' -import { - claudeAuthDiagnostic, - readClaudeCapabilities, - readClaudeFrameString, - readClaudeInit, - readClaudeModels -} from './claude-structured-init-proof' -import { - createClaudeInitDeadline, - requestClaudeInitialization -} from './claude-structured-init-deadline' +import { readClaudeFrameString, readClaudeInit } from './claude-structured-init-proof' import { claudeConfigDirEnvPatch } from './claude-config-dir-pin' import { CLAUDE_SPAWN_TOKEN_ENV, claudeProcessIdentity } from './claude-structured-owner-identity' -import { restoreClaudeStructuredSessionOptions } from './claude-structured-options' import { ClaudePromptRegistry } from './claude-structured-prompt-replies' +import { restoredClaudeStructuredSessionOptions } from './claude-structured-options' import { createClaudeSessionJournalTranslator } from './claude-structured-journal-translation' +import { observeClaudeFastModeFacts } from './claude-structured-session-options' import { - observeClaudeFastModeFacts, - readClaudeSettingsEffort -} from './claude-structured-session-options' -import { - claudeStructuredSessionPublicationOptions, - prepareClaudeStructuredSessionAcquisitionOptions, - readClaudeStructuredSessionSettings -} from './claude-structured-session-acquisition-options' + createClaudeInitProof, + readClaudeStartupFacts, + settleClaudeSessionStartup +} from './claude-structured-session-startup' import { createClaudeSessionPublication } from './claude-structured-session-publication' import { mintClaudeAcquisitionGeneration, @@ -52,8 +39,6 @@ import { createClaudeJournalFailureHandler } from './claude-structured-session-journal-control' -export const CLAUDE_STRUCTURED_INIT_TIMEOUT_MS = 10_000 - export async function acquireClaudeSession({ input, deps, @@ -81,15 +66,16 @@ export async function acquireClaudeSession({ let liveSession: ClaudeSession | null = null let observedLeafUuid: string | null = null, expectedProviderSessionId: string | null = null + // The CLI's own account of why it ended (stderr included): the only reason a user can act on. + let childEnded: Error | null = null // Frames are admitted only after launch resolution proves the provider session // this acquisition owns. Keep the check ahead of every stateful consumer. - const initTimeoutMs = deps.initTimeoutMs ?? CLAUDE_STRUCTURED_INIT_TIMEOUT_MS - const initDeadline = createClaudeInitDeadline(sessionId, initTimeoutMs) + const initProof = createClaudeInitProof() const translator = createClaudeSessionJournalTranslator( input.events, prompts, String(input.fence), - createClaudeJournalFailureHandler({ attempt, initDeadline, callbacks, sessionId }) + createClaudeJournalFailureHandler({ attempt, initProof, callbacks, sessionId }) ) const onMessage = (message: Record): void => { @@ -98,12 +84,12 @@ export async function acquireClaudeSession({ // An init proof for another (or unnamed) provider must fail acquisition // promptly, while ordinary foreign frames stay quarantined silently. if (init || (message.type === 'system' && message.subtype === 'init')) { - initDeadline.reject(new Error('claude provider session expected')) + initProof.reject(new Error('claude provider session expected')) } return } if (init) { - initDeadline.resolve(init) + initProof.resolve(init) // Every turn opens with an init frame naming the model the CLI is actually // running; set_model answers success for a model it never resolves, so this // report is the session's only adoption evidence. @@ -187,14 +173,12 @@ export async function acquireClaudeSession({ canUseTool, onUserDialog, onFault: (error) => { - if (!attempt.published) { - initDeadline.reject(error) - } + childEnded ??= error + initProof.reject(error) }, onExit: (error) => { - if (!attempt.published) { - initDeadline.reject(error) - } + childEnded ??= error + initProof.reject(error) callbacks.handleExit(sessionId, attempt, error) } } @@ -208,89 +192,88 @@ export async function acquireClaudeSession({ deps.now ? { now: deps.now } : {} ) acquisitions.assertCurrent(sessionId, attempt) - initDeadline.start() - const [initialization, init] = await withAgentSessionCreatePhase( - 'init', - input.recordPhase, - () => - Promise.all([ - requestClaudeInitialization(connection, sessionId, initTimeoutMs), - initDeadline.promise - ]) - ) - const models = readClaudeModels(initialization) - callbacks.deliver(attempt, sessionId, () => - callbacks.emit(liveSession, input.events, { type: 'options', sessionId, models }) - ) - initDeadline.clear() - acquisitions.assertCurrent(sessionId, attempt) - if (init.providerSessionId !== launch.providerSessionId) { - throw new Error( - `claude proved session ${init.providerSessionId}, expected ${launch.providerSessionId}` - ) + const emit = (event: Parameters[2]): void => + callbacks.deliver(attempt, sessionId, () => callbacks.emit(liveSession, input.events, event)) + if (connection.pid === undefined) { + // A pid-less spawn always reports its error next; surface that, not the missing pid. + await initProof.promise } - const settings = await readClaudeStructuredSessionSettings(connection, deps.requestTimeoutMs) - const acquisitionOptions = prepareClaudeStructuredSessionAcquisitionOptions({ - settings, - initialization, - inputOptions: input.options, - resumed: launch.resumed - }) - callbacks.deliver(attempt, sessionId, () => - callbacks.emit(liveSession, input.events, { - type: 'auth-diagnostic', - sessionId, - diagnostic: claudeAuthDiagnostic(init, settings) - }) - ) const process = await claudeProcessIdentity( { ...input, pid: connection.pid }, deps.readProcessStartTime - ) + ).catch((error: unknown) => { + // A child that already ended explains why its start time could not be read. + throw childEnded ?? error + }) acquisitions.assertCurrent(sessionId, attempt) if (connection.closed) { - throw new Error(`claude stream-json for session ${sessionId} exited while being acquired`) + throw ( + childEnded ?? + new Error(`claude stream-json for session ${sessionId} exited while being acquired`) + ) } - const publication = await withAgentSessionCreatePhase('publish', input.recordPhase, async () => - createClaudeSessionPublication({ - connection, - init, - initialization, - leafUuid: observedLeafUuid, - turnEndLeafUuid: launch.resumeLeafUuid, - fence: input.fence, - effort: readClaudeSettingsEffort(settings), - ...claudeStructuredSessionPublicationOptions(acquisitionOptions), - resumed: launch.resumed, - prompts, - translator, - events: input.events, - ...(unbindReadingControl ? { unbindReadingControl } : {}), - process, - acquisitionGeneration: mintClaudeAcquisitionGeneration(deps), - options: acquisitionOptions.options, - capabilities: readClaudeCapabilities(init, initialization), - ...(deps.mintLinkId ? { linkId: deps.mintLinkId() } : {}), - observedAt: deps.now?.() ?? Date.now() - }) - ) - const acquired: AgentSessionAcquisition = publication.acquisition - liveSession = publication.session - await withAgentSessionCreatePhase('restore_options', input.recordPhase, () => - restoreClaudeStructuredSessionOptions(liveSession!, deps.requestTimeoutMs) - ) - acquisitions.assertCurrent(sessionId, attempt) + const publication = createClaudeSessionPublication({ + connection, + providerSessionId: launch.providerSessionId, + leafUuid: observedLeafUuid, + turnEndLeafUuid: launch.resumeLeafUuid, + fence: input.fence, + continuesChain: launch.continuesChain, + prompts, + translator, + events: input.events, + ...(unbindReadingControl ? { unbindReadingControl } : {}), + process, + acquisitionGeneration: mintClaudeAcquisitionGeneration(deps), + options: restoredClaudeStructuredSessionOptions(input.options), + ...(deps.mintLinkId ? { linkId: deps.mintLinkId() } : {}), + observedAt: deps.now?.() ?? Date.now() + }) + const session = publication.session + liveSession = session acquisitions.deleteIfCurrent(sessionId, attempt) await withAgentSessionCreatePhase('publish', input.recordPhase, async () => { - sessions.set(sessionId, liveSession!) + sessions.set(sessionId, session) attempt.published = true for (const event of attempt.buffered.splice(0)) { event() } }) - return acquired + session.startup.settled = settleClaudeSessionStartup({ + session, + facts: readClaudeStartupFacts({ + connection, + initProof, + sessionId, + providerSessionId: launch.providerSessionId, + resumesTranscript: launch.resumesTranscript, + inputOptions: input.options, + requestTimeoutMs: deps.requestTimeoutMs, + emit + }), + isCurrent: () => sessions.get(sessionId) === session, + requestTimeoutMs: deps.requestTimeoutMs, + fault: (error) => callbacks.handleExit(sessionId, attempt, error), + onStarted: (options) => + emit({ + type: 'started', + sessionId, + fence: input.fence, + acquisitionGeneration: session.acquisitionGeneration, + ...options + }) + }) + // A child whose exit already reached `handleExit` is not handed over as live: the create + // fails with the CLI's own diagnostic, as one that died before publish does. + if (sessions.get(sessionId) !== session) { + throw ( + exits.get(sessionId)?.error ?? new Error('claude session ended before acquisition returned') + ) + } + // The start applies its facts and restores saved options only after publish, so the child + // is `starting` until `started` says otherwise. + return { ...publication.acquisition, providerChildPhase: 'starting' } } catch (error) { - initDeadline.clear() unbindReadingControl?.() const acquisitionError = await resolveClaudeAcquisitionError({ error, diff --git a/src/main/claude/claude-structured-session-adapter.test.ts b/src/main/claude/claude-structured-session-adapter.test.ts index c4fc4b7ff611..e1699772836c 100644 --- a/src/main/claude/claude-structured-session-adapter.test.ts +++ b/src/main/claude/claude-structured-session-adapter.test.ts @@ -3,17 +3,15 @@ import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { AgentSessionAcquisitionExitUnprovenError, - AgentSessionAcquisitionRefusal, AgentSessionAcquisitionRootExitObservedError } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { ClaudeStreamJsonConnection } from './claude-stream-json-connection' import { ClaudeControlRequestError } from './claude-stream-json-connection' import { CLAUDE_SPAWN_TOKEN_ENV } from './claude-structured-owner-identity' import { encodeClaudeQuestionOptionId } from './claude-structured-prompt-replies' -import { - CLAUDE_STRUCTURED_INIT_TIMEOUT_MS, - type ClaudeStructuredSessionAdapter, - type ClaudeStructuredSessionEvent +import type { + ClaudeStructuredSessionAdapter, + ClaudeStructuredSessionEvent } from './claude-structured-session-adapter' import { acquired, @@ -28,10 +26,6 @@ import { } from './claude-structured-session-test-support' describe('ClaudeStructuredSessionAdapter.acquire', () => { - it('finishes its startup deadline before the paired mobile request deadline', () => { - expect(CLAUDE_STRUCTURED_INIT_TIMEOUT_MS).toBeLessThan(30_000) - }) - it('pins the account and proves init without treating the system-frame uuid as a chain leaf', async () => { const claude = fakeClaude() const events: ClaudeStructuredSessionEvent[] = [] @@ -73,7 +67,7 @@ describe('ClaudeStructuredSessionAdapter.acquire', () => { it('restores persisted model and effort before publishing a reacquired session', async () => { const claude = fakeClaude() - const adapter = adapterFor(claude, { resumed: true }) + const adapter = adapterFor(claude, { resumesTranscript: true, continuesChain: true }) await adapter.acquire({ identity: identityFor(), @@ -146,7 +140,7 @@ describe('ClaudeStructuredSessionAdapter.acquire', () => { list_models: () => [{ value: 'opus', displayName: 'Opus', supportsFastMode: true }] } }) - const adapter = adapterFor(claude, { resumed: true }) + const adapter = adapterFor(claude, { resumesTranscript: true, continuesChain: true }) await adapter.acquire({ identity: identityFor(), @@ -221,7 +215,6 @@ describe('ClaudeStructuredSessionAdapter.acquire', () => { } } }) - const adapter = adapterFor(claude) const input = { identity: identityFor(), fence: 7, @@ -229,7 +222,11 @@ describe('ClaudeStructuredSessionAdapter.acquire', () => { options: { model: 'temporarily-unavailable' } } - await expect(adapter.acquire(input)).rejects.toThrow('claude set_model request timed out') + // Restore runs after publish, so its failure ends the session rather than the create. + await expect(endedAtStartup(claude, input)).resolves.toMatchObject({ + reason: 'claude set_model request timed out', + startupUnproven: true + }) expect(claude.connections[0]?.closeCount).toBe(1) }) @@ -465,7 +462,8 @@ describe('ClaudeStructuredSessionAdapter.acquire', () => { it('resumes the same provider id and refuses an init proof for another session', async () => { const resumedClaude = fakeClaude() const resumed = adapterFor(resumedClaude, { - resumed: true, + resumesTranscript: true, + continuesChain: true, resumeLeafUuid: 'leaf-before' }) const acquisition = await resumed.acquire({ @@ -481,37 +479,32 @@ describe('ClaudeStructuredSessionAdapter.acquire', () => { }) const wrongClaude = fakeClaude({ initSessionId: 'different-session' }) - const wrong = adapterFor(wrongClaude) - await expect( - wrong.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) - ).rejects.toThrow(/expected/) + await expect(endedAtStartup(wrongClaude)).resolves.toMatchObject({ + reason: expect.stringMatching(/expected/), + startupUnproven: true + }) expect(wrongClaude.connections[0].closeCount).toBe(1) }) - it('surfaces a CLI startup failure instead of waiting for the init deadline', async () => { + it('fails the acquire with the CLI diagnostic when the exit lands before the handover', async () => { + // No init delay: the child dies inside the initialize call, before acquire can return it. const claude = fakeClaude({ exitBeforeInit: 'Claude login required' }) - const adapter = adapterFor(claude) + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterFor(claude, {}, events) await expect( adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) ).rejects.toThrow('Claude login required') - expect(claude.connections[0].closeCount).toBe(1) - }) + await adapter.drainObservedExits() - it('closes a silent unauthenticated startup with actionable account guidance', async () => { - const claude = fakeClaude({ initProof: 'none' }) - const adapter = adapterFor(claude, {}, [], [], 20) - - const error = await adapter - .acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) - .catch((cause: unknown) => cause) - - expect(error).toBeInstanceOf(AgentSessionAcquisitionRefusal) - expect(error).toMatchObject({ - message: expect.stringMatching(/selected Claude account is signed in.*CLAUDE_CONFIG_DIR/s) + // The published-then-ended path is the slow-init case in the startup suite. The first-hand + // exit is still reported as it was seen; the host holds no session under it to end. + expect(events.find((event) => event.type === 'ended')).toMatchObject({ + reason: 'Claude login required', + cause: 'unexpected-exit', + startupUnproven: true }) - expect(claude.connections[0].calls[0]).toEqual({ subtype: 'initialize' }) - expect(claude.connections[0].closeCount).toBe(1) + expect(claude.connections[0].closeCount).toBeGreaterThanOrEqual(1) }) it('refuses an unauthenticated initialize response even when SessionStart runs', async () => { @@ -519,15 +512,32 @@ describe('ClaudeStructuredSessionAdapter.acquire', () => { initProof: 'session-start', initAccount: { apiProvider: 'firstParty', tokenSource: 'none' } }) - const adapter = adapterFor(claude) - await expect( - adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) - ).rejects.toThrow(/not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s) + await expect(endedAtStartup(claude)).resolves.toMatchObject({ + reason: expect.stringMatching(/not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s), + startupUnproven: true + }) expect(claude.connections[0].closeCount).toBe(1) }) }) +/** Acquires, then returns the `ended` event the startup failure published. */ +async function endedAtStartup( + claude: ReturnType, + input: Parameters[0] = { + identity: identityFor(), + fence: 7, + spawnToken: 'spawn-9' + }, + launch: Parameters[1] = {} +): Promise { + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = adapterFor(claude, launch, events) + await adapter.acquire(input) + await adapter.drainObservedExits() + return events.find((event) => event.type === 'ended') +} + describe('ClaudeStructuredSessionAdapter acquisition cleanup', () => { /** A start that fails after the child self-exited, with its close verdict scripted. */ function failedStart( @@ -537,7 +547,11 @@ describe('ClaudeStructuredSessionAdapter acquisition cleanup', () => { exitBeforeInit: 'claude stream-json exited (code 1): not logged in', unprovenCloseVerdict }) - return adapterFor(claude) + return adapterFor(claude, { + resumesTranscript: true, + continuesChain: true, + resumeLeafUuid: 'tip' + }) .acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) .catch((error: unknown) => error) } @@ -601,7 +615,12 @@ describe('ClaudeStructuredSessionAdapter acquisition cleanup', () => { it('forgets a retained exit once the session is acquired again', async () => { const options: Parameters[0] = {} const claude = fakeClaude(options) - const adapter = await acquired(claude) + const adapter = adapterFor(claude, { + resumesTranscript: true, + continuesChain: true, + resumeLeafUuid: 'tip' + }) + await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) const first = claude.connections[0] first.handlers.onExit?.(new Error('claude stream-json exited (code 1): crashed')) first.exitVerdict = { root: 'exited', tree: 'unverifiable' } @@ -635,6 +654,40 @@ describe('ClaudeStructuredSessionAdapter acquisition cleanup', () => { ) }) + it('names a settled exit to a late caller until the chat is acquired again', async () => { + const claude = fakeClaude() + const adapter = await acquired(claude) + claude.connections[0].handlers.onExit?.(new Error('claude stream-json exited: not logged in')) + await adapter.drainObservedExits() + expect(() => adapter.readOptions({ sessionId: 'session-1', fence: 7 })).toThrow('not logged in') + + await adapter.acquire({ identity: identityFor(), fence: 8, spawnToken: 'spawn-10' }) + await expect(adapter.releaseAcquisition({ sessionId: 'session-1' })).resolves.toBe(true) + expect(() => adapter.readOptions({ sessionId: 'session-1', fence: 8 })).toThrow( + 'no live claude stream-json session' + ) + }) + + it('forgets an exit the chat was closed over, even one that settles during the close', async () => { + const claude = fakeClaude() + const adapter = await acquired(claude) + const connection = claude.connections[0] + const proof = Promise.withResolvers() + connection.close = vi + .fn() + .mockImplementationOnce(() => proof.promise) + .mockResolvedValue(true) + connection.handlers.onExit?.(new Error('claude stream-json exited: not logged in')) + await tick() + + const closing = adapter.closeSession('session-1') + proof.resolve(true) + await expect(closing).resolves.toBe(true) + expect(() => adapter.readOptions({ sessionId: 'session-1', fence: 7 })).toThrow( + 'no live claude stream-json session' + ) + }) + it('does not report a second release as successful while retained exit evidence is unproven', async () => { const claude = fakeClaude({ unprovenCloseVerdict: { root: 'exited', tree: 'unverifiable' } }) const adapter = await acquired(claude) diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index 6523458465bb..d28b07dc16d3 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -9,9 +9,8 @@ import { dispatchClaudeTurn } from './claude-structured-dispatch' import { StructuredSessionCompaction } from '../native-chat/agent-session-wire/structured-session-compaction' import { releaseClaudeAcquisition } from './claude-structured-acquisition-release' import { acquireClaudeSession } from './claude-structured-session-acquisition' -export { CLAUDE_STRUCTURED_INIT_TIMEOUT_MS } from './claude-structured-session-acquisition' import { supportsClaudeStructuredLocation } from './claude-structured-location-support' -import { setClaudeStructuredOption } from './claude-structured-options' +import { setClaudeStructuredSessionOption } from './claude-structured-options' import { readClaudeStructuredSessionOptions } from './claude-structured-session-options' import { ClaudeAcquisitionRegistry, @@ -21,14 +20,12 @@ import { type ClaudeStructuredSessionAdapterDeps, type ClaudeStructuredSessionEvent } from './claude-structured-session-state' -import { - closeAllClaudeSessions, - closeClaudeSession, - settleClaudeExitedSession -} from './claude-structured-session-close' +import { closeAllClaudeSessions, closeClaudeSession } from './claude-structured-session-close' import { drainClaudeObservedExits, - persistClaudeSessionHandle + observeClaudeSessionExit, + settleClaudeUnexpectedExit, + type ClaudeExitLifecycle } from './claude-structured-session-exit-lifecycle' import type { AgentSessionBackgroundTaskState } from '../../shared/agent-session-wire' import { resolveClaudeProviderHistoryWindow } from './claude-structured-history-window' @@ -55,8 +52,18 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda private readonly sessions = new Map() private readonly acquisitions = new ClaudeAcquisitionRegistry() private readonly exits = new Map() + private readonly settledExitErrors = new Map() + private readonly exitLifecycle: ClaudeExitLifecycle - constructor(private readonly deps: ClaudeStructuredSessionAdapterDeps) {} + constructor(private readonly deps: ClaudeStructuredSessionAdapterDeps) { + this.exitLifecycle = { + sessions: this.sessions, + exits: this.exits, + settledExitErrors: this.settledExitErrors, + deps, + emit: (session, event) => this.emit(session, event) + } + } supportsLocation = supportsClaudeStructuredLocation @@ -66,8 +73,9 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda reason: 'unsupported' }) - acquire = (input: StructuredAgentSessionAcquireInput): Promise => - acquireClaudeSession({ + acquire = (input: StructuredAgentSessionAcquireInput): Promise => { + this.settledExitErrors.delete(input.identity.sessionId) + return acquireClaudeSession({ input, deps: this.deps, sessions: this.sessions, @@ -76,10 +84,13 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda callbacks: { deliver: (attempt, sessionId, event) => this.deliver(attempt, sessionId, event), emit: (session, _events, event) => this.emit(session, event), - handleExit: (sessionId, attempt, error) => this.handleExit(sessionId, attempt, error), - settleExit: (sessionId, exit) => this.settleUnexpectedExit(sessionId, exit) + handleExit: (sessionId, attempt, error) => + observeClaudeSessionExit(this.exitLifecycle, sessionId, attempt, error), + settleExit: (sessionId, exit) => + settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit) } }) + } private deliver(attempt: ClaudeAcquisitionAttempt, sessionId: string, event: () => void): void { if (!attempt.published) { @@ -94,33 +105,6 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } } - private handleExit(sessionId: string, attempt: ClaudeAcquisitionAttempt, error: Error): void { - const session = this.sessions.get(sessionId) - if (!session || session.connection !== attempt.connection) { - return - } - this.sessions.delete(sessionId) - // Re-enter the provider's close ladder before publishing lifecycle recovery. - // An exit callback is root evidence only; the retained tree proof must run - // before the host releases and reacquires this exact child. - const closePromise = session.connection.close().catch(() => false) - const exit: ClaudeSessionExit = { - connection: session.connection, - session, - error, - closePromise - } - this.exits.set(sessionId, exit) - exit.publication = closePromise - .then((proven) => { - if (!proven) { - return undefined - } - return this.settleUnexpectedExit(sessionId, exit) - }) - .catch(() => undefined) - } - /** Resolves once every first-hand exit observed so far has published its * lifecycle event — or has failed its tree proof and stayed indexed for a * retry. Publication trails observation by the close ladder and the @@ -128,44 +112,9 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda * apart without guessing at wall-clock. */ drainObservedExits = (): Promise => drainClaudeObservedExits(this.exits) - /** Lifecycle recovery is published only after the child tree proof is true. */ - private settleUnexpectedExit(sessionId: string, exit: ClaudeSessionExit): Promise { - exit.settlementPromise ??= (async () => { - exit.session.unbindReadingControl?.() - if (this.exits.get(sessionId) !== exit) { - settleClaudeExitedSession(exit.session) - return - } - // Persist the last completed turn before publishing the lifecycle - // event that lets the host release and reacquire this exact child. - await persistClaudeSessionHandle(sessionId, exit.session, this.deps).catch( - (error: unknown) => { - // Recovery still publishes: the record keeps its last durable point, and the loss is logged. - console.warn('[claude-resume-point] exit cursor was not persisted:', { sessionId, error }) - } - ) - if (this.exits.get(sessionId) !== exit) { - settleClaudeExitedSession(exit.session) - return - } - this.exits.delete(sessionId) - const ended: ClaudeStructuredSessionEvent = { - type: 'ended', - sessionId, - reason: exit.error.message, - cause: 'unexpected-exit', - fence: exit.session.fence, - acquisitionGeneration: exit.session.acquisitionGeneration, - observedAt: this.deps.now?.() ?? Date.now() - } - try { - this.emit(exit.session, ended) - } finally { - settleClaudeExitedSession(exit.session) - } - })() - return exit.settlementPromise - } + /** Resolves once a published session's startup has landed or faulted it. */ + drainStartup = (sessionId: string): Promise => + this.sessions.get(sessionId)?.startup.settled ?? Promise.resolve() /** Restart reconciliation reads the transcript a resume replays; these maps track liveness. */ providerHistoryWindow: NonNullable = ( @@ -214,7 +163,9 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } dispatch: StructuredAgentSessionAdapter['dispatch'] = (input) => - dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch) + dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch, (settlement) => + this.deps.onDispatchSettledLate?.({ sessionId: input.sessionId, ...settlement }) + ) compact: NonNullable = (input) => compactClaudeSession(this.session(input.sessionId), this.compactions, input) @@ -257,7 +208,11 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda answerPrompt: StructuredAgentSessionAdapter['answerPrompt'] = (request) => answerClaudeStructuredPrompt({ request, sessions: this.sessions }) setOption: StructuredAgentSessionAdapter['setOption'] = (input) => - setClaudeStructuredOption(this.session(input.sessionId), input, this.deps.requestTimeoutMs) + setClaudeStructuredSessionOption( + this.session(input.sessionId), + input, + this.deps.requestTimeoutMs + ) readOptions = (input: { sessionId: string; fence: number }) => readClaudeStructuredSessionOptions(this.session(input.sessionId), this.deps.requestTimeoutMs) recordsContextUsage = (sessionId: string): boolean => this.sessions.has(sessionId) @@ -272,7 +227,8 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda sessions: this.sessions, acquisitions: this.acquisitions, exits: this.exits, - onExitProven: (sessionId, exit) => this.settleUnexpectedExit(sessionId, exit), + onExitProven: (sessionId, exit) => + settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit), ...(this.deps.persistHandle ? { persistHandle: this.deps.persistHandle } : {}), ...(this.deps.onBackgroundTasksChanged ? { onBackgroundTasksChanged: this.deps.onBackgroundTasksChanged } @@ -280,7 +236,11 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda ...(this.deps.onEvent ? { onEvent: this.deps.onEvent } : {}) }) - closeSession = (sessionId: string): Promise => { + closeSession = (sessionId: string): Promise => + // After the close, not before: releasing an exit still settling settles it on the way. + this.closeSessionProcess(sessionId).finally(() => this.settledExitErrors.delete(sessionId)) + + private closeSessionProcess(sessionId: string): Promise { if (this.exits.has(sessionId)) { return this.releaseAcquisition({ sessionId }) } @@ -308,7 +268,12 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda private session(sessionId: string): ClaudeSession { const session = this.sessions.get(sessionId) if (!session) { - throw new Error(`no live claude stream-json session for ${sessionId}`) + // A child that just exited is named by its own diagnostic, not by its absence. + throw ( + this.exits.get(sessionId)?.error ?? + this.settledExitErrors.get(sessionId) ?? + new Error(`no live claude stream-json session for ${sessionId}`) + ) } return session } diff --git a/src/main/claude/claude-structured-session-close.ts b/src/main/claude/claude-structured-session-close.ts index 0ce448b4e9a0..2d1b1f255d85 100644 --- a/src/main/claude/claude-structured-session-close.ts +++ b/src/main/claude/claude-structured-session-close.ts @@ -19,6 +19,14 @@ import { closeProcessRegistry } from '../../shared/child-process/close-process-r import { retireClaudeDispatchWaiters } from './claude-structured-dispatch' import { settledClaudeTurnEndLeaf } from './claude-structured-resume-point' +/** The root's own exit was seen first-hand; only its descendants went unverified. */ +export function claudeRootExitObserved( + connection: ClaudeStreamJsonConnection | null | undefined +): boolean { + const verdict = connection?.exitVerdict + return verdict?.root === 'exited' && verdict.tree === 'unverifiable' +} + export function claudeAcquisitionCleanupError( connection: ClaudeStreamJsonConnection | null | undefined, cause: unknown @@ -27,7 +35,7 @@ export function claudeAcquisitionCleanupError( if (verdict?.root === 'processless') { return new AgentSessionPreSpawnError(cause) } - return verdict?.root === 'exited' && verdict.tree === 'unverifiable' + return claudeRootExitObserved(connection) ? new AgentSessionAcquisitionRootExitObservedError(cause) : new AgentSessionAcquisitionExitUnprovenError(cause) } diff --git a/src/main/claude/claude-structured-session-exit-lifecycle.ts b/src/main/claude/claude-structured-session-exit-lifecycle.ts index f8a4f6bf8e61..fb1ab1943b4b 100644 --- a/src/main/claude/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/claude-structured-session-exit-lifecycle.ts @@ -1,10 +1,105 @@ import { settledClaudeTurnEndLeaf } from './claude-structured-resume-point' +import { + claudeRootExitObserved, + settleClaudeExitedSession +} from './claude-structured-session-close' +import { failClaudeStartupGate } from './claude-structured-session-startup-gate' import type { + ClaudeAcquisitionAttempt, ClaudeSession, ClaudeSessionExit, - ClaudeStructuredSessionAdapterDeps + ClaudeStructuredSessionAdapterDeps, + ClaudeStructuredSessionEvent } from './claude-structured-session-state' +export type ClaudeExitLifecycle = { + sessions: Map + exits: Map + /** A settled exit's diagnostic, kept for a send admitted before the host heard of the exit. */ + settledExitErrors: Map + deps: Pick + emit: (session: ClaudeSession, event: ClaudeStructuredSessionEvent) => void +} + +export function observeClaudeSessionExit( + lifecycle: ClaudeExitLifecycle, + sessionId: string, + attempt: ClaudeAcquisitionAttempt, + error: Error +): void { + const session = lifecycle.sessions.get(sessionId) + if (!session || session.connection !== attempt.connection) { + return + } + lifecycle.sessions.delete(sessionId) + failClaudeStartupGate(session, error) + // Re-enter the provider's close ladder before publishing lifecycle recovery. + // An exit callback is root evidence only; the retained tree proof must run + // before the host releases and reacquires this exact child. + const closePromise = session.connection.close().catch(() => false) + const exit: ClaudeSessionExit = { + connection: session.connection, + session, + error, + closePromise + } + lifecycle.exits.set(sessionId, exit) + exit.publication = closePromise + .then((proven) => { + // A failed startup keeps the failed-create bar: a first-hand root exit releases it. + const startupFailed = session.startup.state === 'failed' + if (!proven && !(startupFailed && claudeRootExitObserved(session.connection))) { + return undefined + } + return settleClaudeUnexpectedExit(lifecycle, sessionId, exit) + }) + .catch(() => undefined) +} + +/** Lifecycle recovery is published only after the child tree proof is true. */ +export function settleClaudeUnexpectedExit( + lifecycle: ClaudeExitLifecycle, + sessionId: string, + exit: ClaudeSessionExit +): Promise { + const { exits, deps } = lifecycle + exit.settlementPromise ??= (async () => { + exit.session.unbindReadingControl?.() + if (exits.get(sessionId) !== exit) { + settleClaudeExitedSession(exit.session) + return + } + // Persist the last completed turn before publishing the lifecycle + // event that lets the host release and reacquire this exact child. + await persistClaudeSessionHandle(sessionId, exit.session, deps).catch((error: unknown) => { + // Recovery still publishes: the record keeps its last durable point, and the loss is logged. + console.warn('[claude-resume-point] exit cursor was not persisted:', { sessionId, error }) + }) + if (exits.get(sessionId) !== exit) { + settleClaudeExitedSession(exit.session) + return + } + exits.delete(sessionId) + lifecycle.settledExitErrors.set(sessionId, exit.error) + const ended: ClaudeStructuredSessionEvent = { + type: 'ended', + sessionId, + reason: exit.error.message, + cause: 'unexpected-exit', + fence: exit.session.fence, + acquisitionGeneration: exit.session.acquisitionGeneration, + observedAt: deps.now?.() ?? Date.now(), + ...(exit.session.startup.state === 'proven' ? {} : { startupUnproven: true }) + } + try { + lifecycle.emit(exit.session, ended) + } finally { + settleClaudeExitedSession(exit.session) + } + })() + return exit.settlementPromise +} + /** Wait for each first-hand exit's publication, including exits observed while waiting. */ export async function drainClaudeObservedExits( exits: Map diff --git a/src/main/claude/claude-structured-session-journal-control.ts b/src/main/claude/claude-structured-session-journal-control.ts index 3a86c7b35b16..9a4d3611435e 100644 --- a/src/main/claude/claude-structured-session-journal-control.ts +++ b/src/main/claude/claude-structured-session-journal-control.ts @@ -5,7 +5,7 @@ import { import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' import type { ClaudeStreamJsonConnection } from './claude-stream-json-connection' import type { ClaudeJournalTranslator } from './claude-structured-journal-translation' -import type { createClaudeInitDeadline } from './claude-structured-init-deadline' +import type { ClaudeInitProof } from './claude-structured-session-startup' import type { ClaudeAcquisitionAttempt, ClaudeAcquireCallbacks @@ -13,13 +13,13 @@ import type { export function createClaudeJournalFailureHandler(input: { attempt: ClaudeAcquisitionAttempt - initDeadline: ReturnType + initProof: ClaudeInitProof callbacks: ClaudeAcquireCallbacks sessionId: string }): (error: Error) => void { return (error) => { if (!input.attempt.published) { - input.initDeadline.reject(error) + input.initProof.reject(error) return } const connection = input.attempt.connection diff --git a/src/main/claude/claude-structured-session-options.ts b/src/main/claude/claude-structured-session-options.ts index 4ad95223dec2..dbc7959d6e97 100644 --- a/src/main/claude/claude-structured-session-options.ts +++ b/src/main/claude/claude-structured-session-options.ts @@ -202,10 +202,23 @@ export async function readClaudeStructuredSessionOptions( timeoutMs: number | undefined ): Promise { const readMutationSequence = session.optionMutationSequence - const [catalog, settings] = await Promise.all([ - session.connection.supportedModels({ timeoutMs }).catch(() => null), - session.connection.getSettings({ timeoutMs }).catch(() => null) - ]) + // Before startup both requests would wait on initialize; answer from the saved options. + const [catalog, settings] = + session.startup.state === 'proven' + ? await Promise.all([ + session.connection.supportedModels({ timeoutMs }).catch(() => null), + session.connection.getSettings({ timeoutMs }).catch(() => null) + ]) + : [null, null] + observeClaudeSettingsReadback(session, settings, readMutationSequence) + return claudeStructuredSessionOptionsFrom(session, catalog, readMutationSequence) +} + +function observeClaudeSettingsReadback( + session: ClaudeSession, + settings: unknown, + readMutationSequence: number +): void { if (settings !== null && readMutationSequence === session.optionMutationSequence) { const effort = readClaudeSettingsEffort(settings) const fastMode = readClaudeSettingsFastMode(settings) @@ -224,6 +237,17 @@ export async function readClaudeStructuredSessionOptions( session.fastModePerSessionOptIn = perSessionOptIn } } +} + +/** The options as main already holds them, over `catalog`; asks the CLI nothing. Startup's + * settings readback and restore's confirmations are applied by the time a start proves, and + * the SDK answers `list_models` from its initialize result, so a started session's snapshot + * passes that result here rather than paying two round trips for what it already read. */ +export function claudeStructuredSessionOptionsFrom( + session: ClaudeSession, + catalog: unknown[] | null, + readMutationSequence = session.optionMutationSequence +): AgentSessionOptionsResult { const discovered = listedModels(catalog ? { models: catalog } : null) const models = discovered.length > 0 ? discovered : seedModels() const current = readClaudeCurrentModel(session) diff --git a/src/main/claude/claude-structured-session-publication.ts b/src/main/claude/claude-structured-session-publication.ts index bca70c9db76d..1d4228ed837e 100644 --- a/src/main/claude/claude-structured-session-publication.ts +++ b/src/main/claude/claude-structured-session-publication.ts @@ -1,22 +1,23 @@ import type { AgentSessionAcquisition } from '../native-chat/agent-session-wire/structured-agent-session-adapter' -import type { ClaudeInitObservation } from './claude-structured-init-proof' import { claudeProviderHandleLink } from './claude-structured-owner-identity' import type { ClaudePromptRegistry } from './claude-structured-prompt-replies' import type { ClaudeJournalTranslator } from './claude-structured-journal-translation' import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' +import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +/** The session as published at spawn: nothing the CLI reports at init is assumed yet. */ export function createClaudeSessionPublication(input: { connection: ClaudeSession['connection'] - init: ClaudeInitObservation - initialization?: unknown + providerSessionId: string leafUuid: string | null /** The launch's stored leaf: a frame seen before publication is not a completed turn. */ turnEndLeafUuid: string | null fence: number acquisitionGeneration: string - resumed: boolean + /** The record's chain already heads this provider session: the link resumes, never creates. */ + continuesChain: boolean prompts: ClaudePromptRegistry translator: ClaudeJournalTranslator | null events: ClaudeSession['events'] @@ -25,24 +26,14 @@ export function createClaudeSessionPublication(input: { linkId?: string observedAt: number options?: ReadonlyMap - capabilities: readonly string[] - /** Read from `get_settings`; `system/init` never reports an effort. */ - effort: string | null - fastMode: boolean | null - fastModePerSessionOptIn: boolean | null - fastModeState?: ClaudeSession['fastModeState'] - fastModeDisabledReason?: string }): { acquisition: AgentSessionAcquisition; session: ClaudeSession } { - const model = input.init.model - const effort = input.effort - const fastMode = input.fastMode return { acquisition: { process: input.process, link: claudeProviderHandleLink({ - sessionId: input.init.providerSessionId, + sessionId: input.providerSessionId, leafUuid: input.leafUuid, - resumed: input.resumed, + resumed: input.continuesChain, fence: input.fence, ...(input.linkId ? { linkId: input.linkId } : {}), observedAt: input.observedAt @@ -51,7 +42,7 @@ export function createClaudeSessionPublication(input: { }, session: { connection: input.connection, - providerSessionId: input.init.providerSessionId, + providerSessionId: input.providerSessionId, leafUuid: input.leafUuid, turnEndLeafUuid: input.turnEndLeafUuid, fence: input.fence, @@ -61,32 +52,20 @@ export function createClaudeSessionPublication(input: { retiredDispatchWaiters: [], replayContentFallbackBlocked: false, backgroundTasks: new ClaudeBackgroundTaskTracker(), - commands: new ClaudeSlashCommandCatalog(input.init.message, input.initialization), + // Undefined until init: an unread catalog is unavailable, not empty. + commands: new ClaudeSlashCommandCatalog(), dispatchSequence: 0, optionMutationSequence: 0, options: new Map(input.options), - capabilities: input.capabilities, - reportedOptions: { - ...(model ? { model } : {}), - ...(effort ? { effort } : {}), - ...(fastMode !== null ? { fastMode } : {}) - }, - ...(input.fastModeState ? { fastModeState: input.fastModeState } : {}), - ...(input.fastModeDisabledReason - ? { fastModeDisabledReason: input.fastModeDisabledReason } - : {}), - ...(input.fastModePerSessionOptIn !== null - ? { fastModePerSessionOptIn: input.fastModePerSessionOptIn } - : {}), + capabilities: [], + reportedOptions: {}, reportedModelMutation: 0, - confirmedOptions: new Set([ - ...(effort ? ['effort'] : []), - ...(fastMode !== null ? ['fastMode'] : []) - ]), + confirmedOptions: new Set(), restoreSkippedOptions: new Set(), translator: input.translator, events: input.events, - ...(input.unbindReadingControl ? { unbindReadingControl: input.unbindReadingControl } : {}) + ...(input.unbindReadingControl ? { unbindReadingControl: input.unbindReadingControl } : {}), + startup: createClaudeSessionStartupGate() } } } diff --git a/src/main/claude/claude-structured-session-reading-control.test.ts b/src/main/claude/claude-structured-session-reading-control.test.ts index 4f6815548b2d..1084c2c86263 100644 --- a/src/main/claude/claude-structured-session-reading-control.test.ts +++ b/src/main/claude/claude-structured-session-reading-control.test.ts @@ -90,8 +90,16 @@ describe('Claude structured reading control', () => { }) it('unbinds when acquisition fails after the connection opens', async () => { - const claude = fakeClaude({ initProof: 'none' }) - const adapter = adapterFor(claude, {}, [], [], 1) + // The child exits before publish, so only the failed acquisition can release the binding. + const claude = fakeClaude() + const open = claude.openConnection + claude.openConnection = async (launch, handlers = {}) => { + const connection = await open(launch, handlers) + claude.connections[0].closed = true + handlers.onExit?.(new Error('claude stream-json exited (code 1)')) + return connection + } + const adapter = adapterFor(claude) const events = controlledSink() await expect( @@ -101,7 +109,7 @@ describe('Claude structured reading control', () => { spawnToken: 'spawn-9', events: events.sink }) - ).rejects.toThrow('did not finish starting') + ).rejects.toThrow('exited (code 1)') expect(events.unbind).toHaveBeenCalledOnce() }) diff --git a/src/main/claude/claude-structured-session-recovery.test.ts b/src/main/claude/claude-structured-session-recovery.test.ts index 098b796e4857..ef8e0c654695 100644 --- a/src/main/claude/claude-structured-session-recovery.test.ts +++ b/src/main/claude/claude-structured-session-recovery.test.ts @@ -64,7 +64,8 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { claudeConfigDir: '/accounts/claude', providerSessionId: PROVIDER_SESSION_ID, resumeLeafUuid: null, - resumed: false + resumesTranscript: false, + continuesChain: false }), onEvent: (event) => { events.push(event) @@ -302,7 +303,8 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { claudeConfigDir: '/accounts/claude', providerSessionId: PROVIDER_SESSION_ID, resumeLeafUuid: null, - resumed: false + resumesTranscript: false, + continuesChain: false } } return { @@ -312,7 +314,8 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { claudeConfigDir: '/accounts/claude', providerSessionId: PROVIDER_SESSION_ID, resumeLeafUuid: durableLeafUuid, - resumed: true + resumesTranscript: true, + continuesChain: true } }) const persistHandle = vi.fn>( @@ -335,6 +338,7 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { spawnToken: 'spawn-9', events: journalSink }) + await adapter.drainStartup('session-1') const first = claude.connections[0] const oldPrompt = invokeCanUseTool(first, 'Bash', 'permission-retained', 'tool-retained') const oldSession = ( diff --git a/src/main/claude/claude-structured-session-startup-gate.ts b/src/main/claude/claude-structured-session-startup-gate.ts new file mode 100644 index 000000000000..020d625b5257 --- /dev/null +++ b/src/main/claude/claude-structured-session-startup-gate.ts @@ -0,0 +1,155 @@ +// A Claude session is published once its child is spawned, before the CLI has answered +// initialize. Prompts sent in that window are held here and written, in order, once startup +// lands (init facts read and saved options restored), so a first turn never runs under +// defaults the restore was about to replace. A held prompt was never written, so a startup +// that fails rejects it rather than leaving its delivery in doubt. + +import type { AgentSessionDispatchOutcome } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' +import { dispatchWriteFailureReason } from '../../shared/structured-agent-session-dispatch-rejection' +import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' +import { claudeUserMessageWasProvablyUnwritten } from './claude-agent-sdk-user-message-queue' +import { + forgetRetiredWaiter, + forgetWaiter, + retireWaiter +} from './claude-structured-dispatch-waiters' +import type { + ClaudeDispatchWaiter, + ClaudeLateDispatchOutcome, + ClaudeSession +} from './claude-structured-session-state' + +type ClaudeStartupHeldWrite = { + waiter: ClaudeDispatchWaiter + message: Record + settleLate?: (outcome: ClaudeLateDispatchOutcome) => void +} + +export type ClaudeSessionStartupGate = { + state: 'pending' | 'proven' | 'failed' + held: ClaudeStartupHeldWrite[] + /** Held prompts are still being written; later prompts must queue behind them. */ + draining: boolean + failure: Error | null + /** Resolves once startup has landed or faulted the session; never rejects. */ + settled: Promise +} + +export function createClaudeSessionStartupGate(): ClaudeSessionStartupGate { + return { state: 'pending', held: [], draining: false, failure: null, settled: Promise.resolve() } +} + +export function claudeStartupFailureReason(session: ClaudeSession): string | null { + return session.startup.state === 'failed' + ? providerStartupFailureRejection(session.startup.failure ?? undefined) + : null +} + +export function claudeStartupHoldsWrites(session: ClaudeSession): boolean { + return session.startup.state === 'pending' || session.startup.draining +} + +/** Admits a prompt while startup is pending; it is written when `openClaudeStartupGate` runs. */ +export async function holdClaudeStartupWrite( + session: ClaudeSession, + input: { + message: Record + arm: () => { waiter: ClaudeDispatchWaiter } + beforeDispatch?: () => Promise + settleLate?: (outcome: ClaudeLateDispatchOutcome) => void + } +): Promise { + if (input.beforeDispatch) { + try { + await input.beforeDispatch() + } catch (error) { + if (error instanceof AgentSessionPreDispatchError) { + throw error + } + return { state: 'rejected', reason: dispatchWriteFailureReason(error) } + } + } + // Startup may have failed while the admission barrier ran. + const failed = claudeStartupFailureReason(session) + if (failed) { + return { state: 'rejected', reason: failed } + } + const { waiter } = input.arm() + session.startup.held.push({ + waiter, + message: input.message, + ...(input.settleLate ? { settleLate: input.settleLate } : {}) + }) + // Startup finished writing what it held while the barrier ran; nothing else would drain this. + if (!claudeStartupHoldsWrites(session)) { + void drainClaudeStartupWrites(session) + } + return { state: 'admitted' } +} + +export async function openClaudeStartupGate(session: ClaudeSession): Promise { + const gate = session.startup + if (gate.state !== 'pending') { + return + } + gate.state = 'proven' + await drainClaudeStartupWrites(session) +} + +async function drainClaudeStartupWrites(session: ClaudeSession): Promise { + const gate = session.startup + gate.draining = true + try { + for (let held = gate.held.shift(); held; held = gate.held.shift()) { + await writeHeld(session, held) + } + } finally { + gate.draining = false + } +} + +async function writeHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite): Promise { + try { + await session.connection.send(held.message) + } catch (error) { + if (held.waiter.settledUuid) { + return + } + if (claudeUserMessageWasProvablyUnwritten(error)) { + rejectHeld(session, held, dispatchWriteFailureReason(error)) + return + } + // Possibly written: only a replay or the child's exit can settle it now. + retireWaiter(session, held.waiter) + held.waiter.resolve(null) + } +} + +function rejectHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite, reason: string): void { + forgetWaiter(session, held.waiter) + forgetRetiredWaiter(session, held.waiter) + held.waiter.resolve(null) + if (held.waiter.clientMessageId) { + held.settleLate?.({ clientMessageId: held.waiter.clientMessageId, state: 'rejected', reason }) + } +} + +/** Rejects every held prompt with `reason`; true when any was held. */ +export function rejectClaudeStartupWrites(session: ClaudeSession, reason: string): boolean { + const held = session.startup.held.splice(0) + for (const entry of held) { + rejectHeld(session, entry, reason) + } + return held.length > 0 +} + +/** Startup cannot land any more; nothing held was written, so all of it is rejected. */ +export function failClaudeStartupGate(session: ClaudeSession, error: Error): void { + const gate = session.startup + if (gate.state === 'pending') { + gate.state = 'failed' + gate.failure = error + } + rejectClaudeStartupWrites(session, providerStartupFailureRejection(error)) +} diff --git a/src/main/claude/claude-structured-session-startup.test.ts b/src/main/claude/claude-structured-session-startup.test.ts new file mode 100644 index 000000000000..1ff338ac1ccf --- /dev/null +++ b/src/main/claude/claude-structured-session-startup.test.ts @@ -0,0 +1,271 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ClaudeStructuredSessionAdapterDeps } from './claude-structured-session-adapter' +import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' +import { + adapterAtPublishFor, + fakeClaude, + identityFor, + USER_MESSAGE +} from './claude-structured-session-test-support' + +type LateSettlement = Parameters< + NonNullable +>[0] + +const SLOW_INIT_MS = 12_000 + +function startingAdapter(claude: ReturnType): { + adapter: ReturnType + events: ClaudeStructuredSessionEvent[] + late: LateSettlement[] +} { + const events: ClaudeStructuredSessionEvent[] = [] + const late: LateSettlement[] = [] + const adapter = adapterAtPublishFor( + claude, + {}, + events, + [], + undefined, + undefined, + undefined, + (settlement) => late.push(settlement) + ) + return { adapter, events, late } +} + +const ACQUIRE = { identity: identityFor(), fence: 7, spawnToken: 'spawn-9' } +const PROMPT = { sessionId: 'session-1', clientMessageId: 'client-1', body: USER_MESSAGE, fence: 7 } + +describe('Claude structured session publishes before the CLI answers initialize', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('creates a session whose init takes longer than any old deadline, then reports its facts', async () => { + const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) + const { adapter, events } = startingAdapter(claude) + + await expect(adapter.acquire(ACQUIRE)).resolves.toBeDefined() + expect(events.some((event) => event.type === 'options')).toBe(false) + expect(adapter.readCommands('session-1')).toBeUndefined() + + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + + expect(events.find((event) => event.type === 'options')).toMatchObject({ + models: [{ value: 'claude-sonnet' }] + }) + expect(events.some((event) => event.type === 'ended')).toBe(false) + expect(claude.connections[0].closeCount).toBe(0) + await adapter.closeAll() + }) + + it('reports `started` once saved options are restored, before any held prompt is written', async () => { + const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, initModel: 'claude-opus-9' }) + const { adapter, events } = startingAdapter(claude) + const order: string[] = [] + claude.routes.set_model = () => { + order.push('set_model') + return undefined + } + await adapter.acquire({ ...ACQUIRE, options: { model: 'opus' } }) + await adapter.dispatch(PROMPT) + expect(events.some((event) => event.type === 'started')).toBe(false) + + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + + const startedAt = events.findIndex((event) => event.type === 'started') + expect(events[startedAt]).toEqual({ + type: 'started', + sessionId: 'session-1', + fence: 7, + acquisitionGeneration: expect.any(String), + // What the restore just proved, carried so the host never asks the CLI again. + reportedOptions: expect.objectContaining({ model: 'opus' }), + restoreSkippedOptions: [] + }) + // The restore wrote the saved model before `started`, and the held prompt only after it. + expect(order).toEqual(['set_model']) + expect(claude.connections[0].sent).toHaveLength(1) + expect(events.slice(0, startedAt).some((event) => event.type === 'options')).toBe(true) + await adapter.closeAll() + }) + + it('holds a prompt sent before init and writes it once startup lands', async () => { + const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) + const { adapter } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + + await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' }) + expect(claude.connections[0].sent).toEqual([]) + + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + + expect(claude.connections[0].sent).toHaveLength(1) + expect(claude.connections[0].sent[0]).toMatchObject({ type: 'user' }) + await adapter.closeAll() + }) + + it('writes a prompt whose admission barrier was still running when startup landed', async () => { + const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) + const { adapter } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + let passBarrier = (): void => {} + const barrier = new Promise((resolve) => { + passBarrier = resolve + }) + + const dispatched = adapter.dispatch({ ...PROMPT, beforeDispatch: () => barrier }) + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + passBarrier() + + await expect(dispatched).resolves.toEqual({ state: 'admitted' }) + expect(claude.connections[0].sent.filter((message) => message.type === 'user')).toHaveLength(1) + await adapter.closeAll() + }) + + it('ends the session with the exit reason when the CLI dies before init, and rejects held prompts', async () => { + const claude = fakeClaude({ + initDelayMs: SLOW_INIT_MS, + exitBeforeInit: 'claude stream-json exited (code 1): stderr says no' + }) + const { adapter, events, late } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + await adapter.dispatch(PROMPT) + + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + await adapter.drainObservedExits() + + expect(events.find((event) => event.type === 'ended')).toMatchObject({ + reason: 'claude stream-json exited (code 1): stderr says no', + cause: 'unexpected-exit', + startupUnproven: true + }) + expect(late).toEqual([ + expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) + ]) + expect(claude.connections[0].sent).toEqual([]) + expect(claude.connections[0].closeCount).toBe(1) + }) + + it('ends a start whose root exit was seen first-hand even when its descendants are unverifiable', async () => { + const claude = fakeClaude({ + initDelayMs: SLOW_INIT_MS, + exitBeforeInit: 'claude stream-json exited (code 1): stderr says no', + unprovenCloseVerdict: { root: 'exited', tree: 'unverifiable' } + }) + const { adapter, events } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + await adapter.drainObservedExits() + + // A failed start is released on the same evidence a failed create is; a proven-live + // descendant would have answered `tree: 'live'` instead. + expect(events.find((event) => event.type === 'ended')).toMatchObject({ + cause: 'unexpected-exit', + startupUnproven: true + }) + }) + + it('ends an unauthenticated start with sign-in guidance', async () => { + const claude = fakeClaude({ initAccount: { apiProvider: 'firstParty', tokenSource: 'none' } }) + const { adapter, events } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + await adapter.drainStartup('session-1') + await adapter.drainObservedExits() + + expect(events.find((event) => event.type === 'ended')).toMatchObject({ + reason: expect.stringMatching(/not signed in/), + startupUnproven: true + }) + }) + + it('closes a session stopped before init without faulting it or writing held prompts', async () => { + const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) + const { adapter, events, late } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + await adapter.dispatch(PROMPT) + + await expect(adapter.closeSession('session-1')).resolves.toBe(true) + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + + const connection = claude.connections[0] + expect(connection.closeCount).toBe(1) + expect(connection.sent).toEqual([]) + expect(connection.calls.map(({ subtype }) => subtype)).not.toContain('get_settings') + expect(late).toEqual([ + expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) + ]) + expect(events.some((event) => event.type === 'ended' && event.startupUnproven)).toBe(false) + expect(events.some((event) => event.type === 'started')).toBe(false) + }) + + it('withdraws a held prompt when the turn is cancelled before init', async () => { + const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) + const { adapter, late } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + await adapter.dispatch(PROMPT) + + await expect( + adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) + ).resolves.toEqual({ cancelled: true }) + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + await adapter.drainStartup('session-1') + + expect(claude.connections[0].sent).toEqual([]) + expect(late).toEqual([ + expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) + ]) + await adapter.closeAll() + }) + + it('withdraws the prompts still held when Stop lands while startup is writing them', async () => { + const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) + const { adapter, late } = startingAdapter(claude) + await adapter.acquire(ACQUIRE) + const connection = claude.connections[0] + const send = connection.send + let landFirstWrite = (): void => {} + const firstWrite = new Promise((resolve) => { + landFirstWrite = resolve + }) + let writes = 0 + connection.send = async (message, beforeDispatch) => { + writes += 1 + if (writes === 1) { + await firstWrite + } + return send(message, beforeDispatch) + } + await adapter.dispatch(PROMPT) + await adapter.dispatch({ ...PROMPT, clientMessageId: 'client-2' }) + + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) + // Startup has landed and is writing the first held prompt. + expect(writes).toBe(1) + const cancelled = adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) + await vi.advanceTimersByTimeAsync(0) + landFirstWrite() + await vi.advanceTimersByTimeAsync(5_000) + await adapter.drainStartup('session-1') + + expect(connection.sent.filter((message) => message.type === 'user')).toHaveLength(1) + expect(late).toContainEqual( + expect.objectContaining({ clientMessageId: 'client-2', state: 'rejected' }) + ) + // Stop withdrew something, so it answers as a cancel whatever the interrupt made of the turn. + await expect(cancelled).resolves.toEqual({ cancelled: true }) + await adapter.closeAll() + }) +}) diff --git a/src/main/claude/claude-structured-session-startup.ts b/src/main/claude/claude-structured-session-startup.ts new file mode 100644 index 000000000000..7c59c94fcb4c --- /dev/null +++ b/src/main/claude/claude-structured-session-startup.ts @@ -0,0 +1,196 @@ +// What Claude reports at initialize, read after the session is already published. None of it +// gates the create: a slow start is still a start, and every way it can fail (exit, auth, +// a foreign session id) faults the published session through its exit path. + +import type { + StructuredAgentSessionAcquireInput, + StructuredAgentSessionStartedEvent +} from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { ClaudeStreamJsonConnection } from './claude-stream-json-connection' +import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' +import { + claudeAuthDiagnostic, + claudeInitializationAuthError, + readClaudeCapabilities, + readClaudeModels, + type ClaudeInitObservation +} from './claude-structured-init-proof' +import { restoreClaudeStructuredSessionOptions } from './claude-structured-options' +import { + claudeStructuredSessionPublicationOptions, + prepareClaudeStructuredSessionAcquisitionOptions, + readClaudeStructuredSessionSettings +} from './claude-structured-session-acquisition-options' +import { + claudeStructuredSessionOptionsFrom, + readClaudeSettingsEffort +} from './claude-structured-session-options' +import { + failClaudeStartupGate, + openClaudeStartupGate +} from './claude-structured-session-startup-gate' +import type { ClaudeSession, ClaudeStructuredSessionEvent } from './claude-structured-session-state' + +export type ClaudeInitProof = { + promise: Promise + resolve: (init: ClaudeInitObservation) => void + reject: (error: Error) => void +} + +export function createClaudeInitProof(): ClaudeInitProof { + let resolve = (_init: ClaudeInitObservation): void => {} + let reject = (_error: Error): void => {} + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + void promise.catch(() => {}) + return { promise, resolve, reject } +} + +export type StructuredAgentSessionStartedOptions = Pick< + StructuredAgentSessionStartedEvent, + 'reportedOptions' | 'restoreSkippedOptions' +> + +export type ClaudeStartupFacts = { + init: ClaudeInitObservation + initialization: unknown + settings: unknown + prepared: ReturnType +} + +/** Settles on the CLI's answers or on its exit; there is no startup timer. */ +export async function readClaudeStartupFacts(input: { + connection: ClaudeStreamJsonConnection + initProof: ClaudeInitProof + sessionId: string + providerSessionId: string + resumesTranscript: boolean + inputOptions: StructuredAgentSessionAcquireInput['options'] + requestTimeoutMs: number | undefined + emit: (event: ClaudeStructuredSessionEvent) => void +}): Promise { + const [initialization, init] = await Promise.all([ + input.connection.initializationResult().then((result) => { + const authError = claudeInitializationAuthError(result) + if (authError) { + throw authError + } + return result + }), + input.initProof.promise + ]) + if (input.connection.closed) { + throw new Error('claude session closed before startup completed') + } + input.emit({ + type: 'options', + sessionId: input.sessionId, + models: readClaudeModels(initialization) + }) + if (init.providerSessionId !== input.providerSessionId) { + throw new Error( + `claude proved session ${init.providerSessionId}, expected ${input.providerSessionId}` + ) + } + const settings = await readClaudeStructuredSessionSettings( + input.connection, + input.requestTimeoutMs + ) + input.emit({ + type: 'auth-diagnostic', + sessionId: input.sessionId, + diagnostic: claudeAuthDiagnostic(init, settings) + }) + return { + init, + initialization, + settings, + prepared: prepareClaudeStructuredSessionAcquisitionOptions({ + settings, + initialization, + inputOptions: input.inputOptions, + resumesTranscript: input.resumesTranscript + }) + } +} + +function applyClaudeStartupFacts(session: ClaudeSession, facts: ClaudeStartupFacts): void { + const { init, initialization, settings, prepared } = facts + const effort = readClaudeSettingsEffort(settings) + const published = claudeStructuredSessionPublicationOptions(prepared) + // A turn's own init frame may already have reported the running model. + if (init.model && session.reportedOptions.model === undefined) { + session.reportedOptions.model = init.model + session.reportedModelMutation = session.optionMutationSequence + } + if (effort) { + session.reportedOptions.effort = effort + session.confirmedOptions.add('effort') + } + if (published.fastMode !== null) { + session.reportedOptions.fastMode = published.fastMode + session.confirmedOptions.add('fastMode') + } + if (published.fastModePerSessionOptIn !== null) { + session.fastModePerSessionOptIn = published.fastModePerSessionOptIn + } + session.fastModeState ??= published.fastModeState + session.fastModeDisabledReason ??= published.fastModeDisabledReason + session.options = prepared.options + session.capabilities = readClaudeCapabilities(init, initialization) + // A catalog frame that streamed in after publish is newer than the initialize answer. + if (session.commands.commands === undefined) { + session.commands = new ClaudeSlashCommandCatalog(init.message, initialization) + } + session.events?.publish() +} + +/** Applies startup facts to the published session, restores saved options, then releases + * held prompts. Any failure faults the session so the user sees why it never started. */ +export async function settleClaudeSessionStartup(input: { + session: ClaudeSession + facts: Promise + isCurrent: () => boolean + requestTimeoutMs: number | undefined + fault: (error: Error) => void + /** Startup has proven; `options` is what the child now reports, snapshotted from memory. */ + onStarted: (options: StructuredAgentSessionStartedOptions) => void +}): Promise { + const { session } = input + const superseded = (): boolean => { + if (input.isCurrent()) { + return false + } + failClaudeStartupGate(session, new Error('claude session closed before startup completed')) + return true + } + try { + const facts = await input.facts + if (superseded()) { + return + } + applyClaudeStartupFacts(session, facts) + await restoreClaudeStructuredSessionOptions(session, input.requestTimeoutMs) + if (!superseded()) { + input.onStarted({ + // `list_models` is answered from this same initialize result, so nothing is re-read. + reportedOptions: claudeStructuredSessionOptionsFrom( + session, + readClaudeModels(facts.initialization) + ).current, + restoreSkippedOptions: [...session.restoreSkippedOptions] + }) + await openClaudeStartupGate(session) + } + } catch (caught) { + const error = caught instanceof Error ? caught : new Error(String(caught)) + // A close or exit that already ended startup owns how the session ends. + const endedElsewhere = session.startup.state !== 'pending' + failClaudeStartupGate(session, error) + if (!endedElsewhere && input.isCurrent()) { + input.fault(error) + } + } +} diff --git a/src/main/claude/claude-structured-session-state.ts b/src/main/claude/claude-structured-session-state.ts index c333f629582e..3c6a8413a917 100644 --- a/src/main/claude/claude-structured-session-state.ts +++ b/src/main/claude/claude-structured-session-state.ts @@ -3,6 +3,7 @@ import type { AgentSessionJournalIdentity } from '../../shared/agent-session-journal-types' import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import type { StructuredAgentSessionStartedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { ClaudeStreamJsonConnection, openClaudeStreamJsonConnection @@ -18,6 +19,7 @@ import type { } from '../../shared/agent-session-wire' import type { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import type { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' +import type { ClaudeSessionStartupGate } from './claude-structured-session-startup-gate' export type ClaudeAuthDiagnostic = { apiKeySourceConfigured: boolean @@ -52,6 +54,8 @@ export type ClaudeStructuredSessionEvent = fence: number } | { type: 'auth-diagnostic'; sessionId: string; diagnostic: ClaudeAuthDiagnostic } + /** Startup facts applied and saved options restored; held prompts are about to be written. */ + | StructuredAgentSessionStartedEvent | { type: 'ended' sessionId: string @@ -63,6 +67,8 @@ export type ClaudeStructuredSessionEvent = settlementRetryRequired?: boolean /** Host clock when the end was observed. */ observedAt?: number + /** The child ended before proving startup, so reacquiring would repeat the same start. */ + startupUnproven?: true } export type ClaudeLateDispatchOutcome = @@ -89,7 +95,6 @@ export type ClaudeStructuredSessionAdapterDeps = { mintAcquisitionGeneration?: () => string now?: () => number requestTimeoutMs?: number - initTimeoutMs?: number persistHandle?: (input: { sessionId: string providerSessionId: string @@ -173,6 +178,8 @@ export type ClaudeSession = { translator: ClaudeJournalTranslator | null events: StructuredAgentSessionEventSink | undefined unbindReadingControl?: () => void + /** Published at spawn; init facts, option restore and queued prompts land when startup does. */ + startup: ClaudeSessionStartupGate } export function mintClaudeAcquisitionGeneration(deps: ClaudeStructuredSessionAdapterDeps): string { diff --git a/src/main/claude/claude-structured-session-test-support.ts b/src/main/claude/claude-structured-session-test-support.ts index 6b65578b1a0a..a951cd059ec1 100644 --- a/src/main/claude/claude-structured-session-test-support.ts +++ b/src/main/claude/claude-structured-session-test-support.ts @@ -57,6 +57,8 @@ export function fakeClaude( /** What `get_context_usage` answers; defaults to an empty, unusable report. */ contextUsage?: unknown exitBeforeInit?: string + /** Host-clock delay before the CLI answers initialize, as on a loaded machine. */ + initDelayMs?: number settings?: unknown replayUuid?: string | null replayUuids?: (string | null)[] @@ -89,9 +91,14 @@ export function fakeClaude( resumeReading: () => {}, initializationResult: async () => { connection.calls.push({ subtype: 'initialize' }) + if (options.initDelayMs !== undefined) { + await new Promise((resolve) => setTimeout(resolve, options.initDelayMs)) + } if (options.exitBeforeInit) { + connection.closed = true handlers.onExit?.(new Error(options.exitBeforeInit)) - return { models: [] } + // The SDK rejects pending control requests once the transport ends. + throw new Error('Query closed before response received') } if (options.initProof === 'session-start') { handlers.onMessage?.({ @@ -202,12 +209,27 @@ export function fakeClaude( return { connections, openConnection, routes } } +/** Acquisition resolves only once startup has landed, as suites written before + * publish-first expect; `adapterAtPublishFor` observes the published window itself. */ export function adapterFor( + ...args: Parameters +): ClaudeStructuredSessionAdapter { + const adapter = adapterAtPublishFor(...args) + const acquire = adapter.acquire + adapter.acquire = async (input) => { + const acquisition = await acquire(input) + await adapter.drainStartup(input.identity.sessionId) + return acquisition + } + return adapter +} + +export function adapterAtPublishFor( claude: ReturnType, launch: Partial = {}, events: ClaudeStructuredSessionEvent[] = [], persistedHandles: unknown[] = [], - initTimeoutMs?: number, + requestTimeoutMs?: number, persistHandle?: ClaudeStructuredSessionAdapterDeps['persistHandle'], onBackgroundTasksChanged?: ClaudeStructuredSessionAdapterDeps['onBackgroundTasksChanged'], onDispatchSettledLate?: ClaudeStructuredSessionAdapterDeps['onDispatchSettledLate'] @@ -220,14 +242,15 @@ export function adapterFor( claudeConfigDir: '/accounts/claude', providerSessionId: PROVIDER_SESSION_ID, resumeLeafUuid: null, - resumed: false, + resumesTranscript: false, + continuesChain: false, ...launch }), onEvent: (event) => events.push(event), openConnection: claude.openConnection, readProcessStartTime: async () => 1_700_000_000_000, now: () => 1_700_000_000_500, - ...(initTimeoutMs === undefined ? {} : { initTimeoutMs }), + ...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }), persistHandle: persistHandle ?? (async (handle) => { diff --git a/src/main/claude/claude-structured-turn-resume-point.test.ts b/src/main/claude/claude-structured-turn-resume-point.test.ts index ab7074611972..4cb72eaa53a6 100644 --- a/src/main/claude/claude-structured-turn-resume-point.test.ts +++ b/src/main/claude/claude-structured-turn-resume-point.test.ts @@ -65,7 +65,8 @@ async function liveOwner( claudeConfigDir: '/accounts/claude', providerSessionId: PROVIDER_SESSION_ID, resumeLeafUuid: 'resumed-at', - resumed: true + resumesTranscript: true, + continuesChain: true }), onEvent: (event) => events.push(event), openConnection: claude.openConnection, diff --git a/src/main/claude/claude-tui-resume-real-binary.integration.test.ts b/src/main/claude/claude-tui-resume-real-binary.integration.test.ts index f8822505c5a5..ff525f640237 100644 --- a/src/main/claude/claude-tui-resume-real-binary.integration.test.ts +++ b/src/main/claude/claude-tui-resume-real-binary.integration.test.ts @@ -189,7 +189,8 @@ describe.skipIf(!claudeAuthenticated)('real Claude TUI resume proof', () => { claudeConfigDir, providerSessionId, resumeLeafUuid: null, - resumed: false + resumesTranscript: false, + continuesChain: false }), onEvent: (event) => events.push(event), onDispatchSettledLate: (settlement) => settlements.push(settlement), diff --git a/src/main/codex/codex-structured-session-close.ts b/src/main/codex/codex-structured-session-close.ts index 2058f86ce855..1ed8dcd385ce 100644 --- a/src/main/codex/codex-structured-session-close.ts +++ b/src/main/codex/codex-structured-session-close.ts @@ -7,7 +7,7 @@ import { type CodexStructuredSessionAdapterDeps, type CodexStructuredSessionEvent } from './codex-structured-session-state' -import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' export function handleCodexSessionExit(input: { sessions: Map @@ -25,7 +25,7 @@ export function handleCodexSessionExit(input: { return false } session.exitObservedAt ??= Date.now() - const event: StructuredAgentSessionLifecycleEvent = { + const event: StructuredAgentSessionEndedEvent = { type: 'ended', sessionId: input.sessionId, reason: input.error.message, diff --git a/src/main/codex/codex-structured-session-state.ts b/src/main/codex/codex-structured-session-state.ts index c890df0be582..9b8659c1e1b6 100644 --- a/src/main/codex/codex-structured-session-state.ts +++ b/src/main/codex/codex-structured-session-state.ts @@ -14,7 +14,7 @@ import type { AgentSessionBackgroundTaskState } from '../../shared/agent-session import type { CodexBackgroundTaskTracker } from './codex-background-task-tracker' import type { CodexJournalTranslator } from './codex-structured-journal-translation' import type { CodexTurnProcessSnapshot } from './codex-structured-turn-processes' -import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { CodexStructuredPermissionPolicy } from './codex-structured-permission-policy' export type CodexStructuredLaunch = { @@ -51,7 +51,7 @@ export type CodexStructuredSessionEvent = codexItemId: string promptKey: string } - | StructuredAgentSessionLifecycleEvent + | StructuredAgentSessionEndedEvent /** Translator-only compatibility for callers that do not participate in host recovery. */ | { type: 'ended'; sessionId: string; reason: string; observedAt?: number } diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index 006cc4ffa383..c4138ba22418 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -29,3 +29,29 @@ export async function markJournalPendingSubmissionsUnknown( } return unresolved.map((entry) => entry.clientMessageId) } + +/** Settles every submission a child that never proved its start left unanswered as `rejected`: + * such a child accepted nothing, so each is provably unwritten and safe to send again. */ +export async function rejectJournalPendingSubmissions( + journal: AgentSessionJournal, + fence: number, + reason: string +): Promise { + const unwritten = journal + .submissions() + .filter( + (entry) => + entry.dispatchState === 'pending' || + (entry.dispatchState === 'unknown' && entry.recovered !== true) + ) + for (const entry of unwritten) { + await journal.resolveDispatch({ + clientMessageId: entry.clientMessageId, + state: 'rejected', + reason, + fence, + recovered: true + }) + } + return unwritten.map((entry) => entry.clientMessageId) +} diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index ef0cc70b1a66..ec445cb70048 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -26,7 +26,10 @@ import type { JournalReplacementItem } from './journal-epoch-replacement' import { readJournalSince } from './journal-cursor' import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open' import { journalDatabaseFile } from './journal-paths' -import { markJournalPendingSubmissionsUnknown } from './journal-pending-submission-recovery' +import { + markJournalPendingSubmissionsUnknown, + rejectJournalPendingSubmissions +} from './journal-pending-submission-recovery' import { applyJournalRow, createJournalReducerState, @@ -289,6 +292,11 @@ export class AgentSessionJournal { return markJournalPendingSubmissionsUnknown(this, fence, reason) } + /** Reject unanswered sends after an owner that never proved its start ended: none was written. */ + async rejectPendingSubmissions(fence: number, reason: string): Promise { + return rejectJournalPendingSubmissions(this, fence, reason) + } + /** The escape hatch for corruption, an unreconcilable prefix, a forked handle, * and an unreadable schema. It invalidates every cursor; clients reload. */ async rollEpoch(reason: AgentJournalEpochReason, fence: number): Promise { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts index 78bf7cf2808f..75dd5f95bfd9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts @@ -419,7 +419,10 @@ describe('structured session acquisition options', () => { now: () => NOW, onAttached: () => {} }) - ).rejects.toThrow('model list unavailable') + ).resolves.toEqual({ + ok: false, + refusal: { code: 'agent_session_operation_invalid', message: 'model list unavailable' } + }) expect(releaseAcquisition).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.ownerProcess).toBeNull() }) @@ -508,9 +511,16 @@ describe('structured session acquisition options', () => { onAttached: () => {} }) - await expect(perform(store, CREATE_OPERATION, null)).rejects.toThrow( - exitProven ? injected.message : 'agent_session_acquisition_exit_unproven' - ) + // A proven exit before the journal opens is answered once, as the refusal its replay gives. + const failed = perform(store, CREATE_OPERATION, null) + await (exitProven && failurePoint !== 'journal' + ? expect(failed).resolves.toEqual({ + ok: false, + refusal: { code: 'agent_session_operation_invalid', message: injected.message } + }) + : expect(failed).rejects.toThrow( + exitProven ? injected.message : 'agent_session_acquisition_exit_unproven' + )) const reopened = await AgentSessionRecordStore.open({ directory: storeDir, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition.ts index 8d8426da4374..7968176a5a9a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition.ts @@ -3,7 +3,8 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { AgentSessionPreSpawnError, isAgentSessionPreSpawnError, - rethrowAfterAgentSessionAcquisitionCleanup + rethrowAfterAgentSessionAcquisitionCleanup, + type StructuredAgentSessionProviderChildPhase } from './structured-agent-session-adapter' import { journalIdentityFor } from './structured-agent-session-attach' import type { AttachFlowInput } from './structured-agent-session-attach-flow' @@ -15,7 +16,11 @@ import { withAgentSessionCreatePhase } from '../../observability/agent-session-i export async function acquireOwner( input: AttachFlowInput, record: AgentSessionRecord -): Promise<{ record: AgentSessionRecord; acquisitionGeneration: string | null }> { +): Promise<{ + record: AgentSessionRecord + acquisitionGeneration: string | null + providerChildPhase: StructuredAgentSessionProviderChildPhase +}> { const fence = record.lease.runtimeFence const spawnToken = record.lease.reservedSpawnToken if (!spawnToken) { @@ -44,14 +49,20 @@ export async function acquireOwner( ...(input.eventSink ? { events: input.eventSink } : {}), ...(input.recordPhase ? { recordPhase: input.recordPhase } : {}) }) - const options = await withAgentSessionCreatePhase('restore_options', input.recordPhase, () => - readNativeSessionOptions({ - adapter: input.adapter, - sessionId: record.sessionId, - fence, - ...(record.options ? { priorOptions: record.options } : {}) - }) - ) + const providerChildPhase = acquired.providerChildPhase ?? 'ready' + // A starting child has proven nothing: the record keeps the reservation's saved options as + // intent, never a catalog guess, and the `started` event persists what the child reports. + const options = + providerChildPhase === 'starting' + ? undefined + : await withAgentSessionCreatePhase('restore_options', input.recordPhase, () => + readNativeSessionOptions({ + adapter: input.adapter, + sessionId: record.sessionId, + fence, + ...(record.options ? { priorOptions: record.options } : {}) + }) + ) if (record.lease.ownerProcess === null) { await input.store.commitProcessIdentity({ sessionId: record.sessionId, @@ -71,7 +82,8 @@ export async function acquireOwner( }) return { record: proved, - acquisitionGeneration: acquired.acquisitionGeneration ?? null + acquisitionGeneration: acquired.acquisitionGeneration ?? null, + providerChildPhase } } catch (error) { if (isAgentSessionPreSpawnError(error)) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.test.ts index 77cce9153f51..93f9b9037df7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.test.ts @@ -1,15 +1,31 @@ import { describe, expect, it, vi } from 'vitest' import { + AgentSessionAcquisitionExitProvenError, AgentSessionAcquisitionExitUnprovenError, + AgentSessionAcquisitionRefusal, AgentSessionAcquisitionRootExitObservedError, rethrowAfterAgentSessionAcquisitionCleanup } from './structured-agent-session-adapter' describe('failed agent-session acquisition cleanup', () => { - it('preserves the acquisition failure after proven cleanup', async () => { + it('names a failure exit-proven after proven cleanup, keeping its diagnostic and cause', async () => { const cause = new Error('proof failed') + const thrown = await rethrowAfterAgentSessionAcquisitionCleanup( + { releaseAcquisition: vi.fn(async () => true) }, + 'session-1', + cause + ).catch((error: unknown) => error) + + expect(thrown).toBeInstanceOf(AgentSessionAcquisitionExitProvenError) + expect(thrown).toMatchObject({ message: 'proof failed', cause }) + }) + it.each([ + ['a refusal', new AgentSessionAcquisitionRefusal('not signed in')], + ['a root exit', new AgentSessionAcquisitionRootExitObservedError(new Error('exited'))], + ['a host store code', new Error('agent_session_checkpoint_stale')] + ])('keeps %s that already names its verdict after proven cleanup', async (_label, cause) => { await expect( rethrowAfterAgentSessionAcquisitionCleanup( { releaseAcquisition: vi.fn(async () => true) }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index 2360a50b3a93..d8407cd2ad33 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -30,6 +30,7 @@ import type { AgentSessionThreadGoalChange, AgentSessionWireRefusalCode } from '../../../shared/agent-session-wire' +import { isAgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' import type { AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' @@ -65,6 +66,15 @@ export class AgentSessionAcquisitionRootExitObservedError extends Error { } } +/** The provider child failed and cleanup proved its whole tree gone. As with a root exit, the + * provider's own diagnostic is the message. */ +export class AgentSessionAcquisitionExitProvenError extends Error { + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }) + this.name = 'AgentSessionAcquisitionExitProvenError' + } +} + export class AgentSessionAcquisitionExitUnprovenError extends Error { constructor(cause: unknown) { super('agent_session_acquisition_exit_unproven', { cause }) @@ -80,6 +90,8 @@ export type AgentSessionAcquisition = { /** Host-local identity for this exact provider child, distinct even when the durable fence is * reused by a superseding acquisition. */ acquisitionGeneration?: string + /** Absent means `ready`: the adapter proved startup before answering. */ + providerChildPhase?: StructuredAgentSessionProviderChildPhase } /** Acquisition failed with first-hand proof that no provider process existed. */ @@ -108,7 +120,7 @@ export type AgentSessionDispatchOutcome = /** The call did not settle. Never re-send on the user's behalf. */ | { state: 'unknown'; reason: string } -export type StructuredAgentSessionLifecycleEvent = { +export type StructuredAgentSessionEndedEvent = { type: 'ended' sessionId: string reason: string @@ -119,8 +131,32 @@ export type StructuredAgentSessionLifecycleEvent = { observedAt?: number /** Translator could not admit terminal rows; host recovery must append its bounded fallback. */ settlementRetryRequired?: boolean + /** The provider ended before it finished starting, so resuming it would repeat the failure. */ + startupUnproven?: true +} + +/** The child a publish-first acquire handed over has now proven its start: startup facts applied + * and saved options restored. What it reports from here on is fact, not a catalog guess. */ +export type StructuredAgentSessionStartedEvent = { + type: 'started' + sessionId: string + fence: number + acquisitionGeneration: string + /** What the child proved, snapshotted by the adapter from what startup already read. The host + * handles this inside the session's serialized step, so it must not ask the CLI. */ + reportedOptions: AgentSessionOptionsResult['current'] + /** Saved options the restore could not apply; the host drops them rather than persist them. */ + restoreSkippedOptions: readonly string[] } +export type StructuredAgentSessionLifecycleEvent = + | StructuredAgentSessionEndedEvent + | StructuredAgentSessionStartedEvent + +/** Whether the provider child behind an acquisition has proven its start. A publish-first + * acquire hands over a `starting` child and the `started` lifecycle event flips it. */ +export type StructuredAgentSessionProviderChildPhase = 'starting' | 'ready' + export type StructuredAgentSessionAcquireInput = { identity: AgentSessionJournalIdentity fence: number @@ -286,7 +322,19 @@ export async function rethrowAfterAgentSessionAcquisitionCleanup( ) } if (released) { - throw cause + throw provenExitAcquisitionFailure(cause) } throw new AgentSessionAcquisitionExitUnprovenError(cause) } + +/** A failure whose child cleanup proved gone. One that already names its own verdict — a + * refusal, a typed exit proof, or a host store code — keeps it. */ +function provenExitAcquisitionFailure(cause: unknown): unknown { + const classified = + cause instanceof AgentSessionAcquisitionRefusal || + cause instanceof AgentSessionAcquisitionRootExitObservedError || + cause instanceof AgentSessionAcquisitionExitUnprovenError || + isAgentSessionPreSpawnError(cause) || + (cause instanceof Error && isAgentSessionWireRefusalCode(cause.message)) + return classified ? cause : new AgentSessionAcquisitionExitProvenError(cause) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts index 9fcc99e41b03..d12dd8eae373 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts @@ -1,10 +1,11 @@ import { settlePostAcquisitionAttachFailure } from './structured-agent-session-attach-failure' import { - AgentSessionAcquisitionExitUnprovenError, - AgentSessionAcquisitionRootExitObservedError, - AgentSessionAcquisitionRefusal, - isAgentSessionPreSpawnError, - type StructuredAgentSessionAdapter + failedAcquisitionRefusal, + failedAcquisitionSettlement +} from './structured-agent-session-failed-create-refusal' +import type { + StructuredAgentSessionAdapter, + StructuredAgentSessionProviderChildPhase } from './structured-agent-session-adapter' // The host supplies owner authority; this flow reserves, proves, and publishes the session. @@ -56,7 +57,8 @@ export type AttachFlowInput = { onAttached: ( attached: AttachedJournal, acquisitionGeneration: string | null, - acquiredOwner: boolean + acquiredOwner: boolean, + providerChildPhase: StructuredAgentSessionProviderChildPhase ) => Promise | void /** Host-owned provider sink, bound to the journal inside `onAttached`. */ eventSink?: StructuredAgentSessionEventSink @@ -92,6 +94,7 @@ export async function performAttach( let record: AgentSessionRecord let acquisitionGeneration: string | null = null let acquiredOwner = false + let providerChildPhase: StructuredAgentSessionProviderChildPhase = 'ready' let reservedRecord: AgentSessionRecord | null = null let unsupportedReservationSettlementAttempted = false let replayed = false @@ -161,37 +164,13 @@ export async function performAttach( ) record = acquired.record acquisitionGeneration = acquired.acquisitionGeneration + providerChildPhase = acquired.providerChildPhase acquiredOwner = true } } catch (error) { const spawnToken = reservedRecord?.lease.reservedSpawnToken if (reservedRecord && spawnToken && !unsupportedReservationSettlementAttempted) { // Settle processless proof and failed operation atomically. - const exitProof = isAgentSessionPreSpawnError(error) - ? 'processless' - : error instanceof AgentSessionAcquisitionExitUnprovenError - ? 'unproven' - : error instanceof AgentSessionAcquisitionRootExitObservedError - ? 'root-exit-observed' - : 'exit-proven' - const outcome = - error instanceof AgentSessionAcquisitionExitUnprovenError - ? { - status: 'failed' as const, - code: 'agent_session_ownership_unknown', - message: error.message - } - : error instanceof AgentSessionAcquisitionRefusal - ? { - status: 'failed' as const, - code: error.code, - message: error.message - } - : { - status: 'failed' as const, - code: 'agent_session_operation_invalid', - message: error instanceof Error ? error.message : String(error) - } try { await store.settleFailedAcquisition({ sessionId, @@ -199,8 +178,7 @@ export async function performAttach( spawnToken, callerKey: input.callerKey, operationId: params.envelope.clientOperationId, - outcome, - exitProof, + ...failedAcquisitionSettlement(error), now: input.now() }) } catch (settlementError) { @@ -210,17 +188,16 @@ export async function performAttach( ) } } - if (error instanceof AgentSessionAcquisitionRefusal) { - return { ok: false, refusal: { code: error.code, message: error.message } } - } - return { - ok: false, - refusal: classifyStoreFailure( - error, - store.getRecord(sessionId)?.lease.runtimeFence ?? null, - store.getRecord(sessionId) - ) - } + return ( + failedAcquisitionRefusal(error) ?? { + ok: false, + refusal: classifyStoreFailure( + error, + store.getRecord(sessionId)?.lease.runtimeFence ?? null, + store.getRecord(sessionId) + ) + } + ) } let attached: AttachedJournal @@ -234,7 +211,7 @@ export async function performAttach( providerHistoryWindow }) await importAdoptedTranscript(params, attached, record, preparedTranscript.items) - await input.onAttached(attached, acquisitionGeneration, acquiredOwner) + await input.onAttached(attached, acquisitionGeneration, acquiredOwner, providerChildPhase) await store.recordOperationOutcome({ callerKey: input.callerKey, operationId: params.envelope.clientOperationId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index 2ceb88625711..5482826d0725 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -15,11 +15,13 @@ import type { } from '../../../shared/agent-session-wire' import type { AgentSessionAttachParams } from './structured-agent-session-attach' import { performAttach } from './structured-agent-session-attach-flow' +import { stampFailedCreateOwnerVerdict } from './structured-agent-session-failed-create-refusal' import { pinnedAgentSessionLaunchArgs, pinnedAgentSessionLaunchEnv } from './structured-agent-session-launch-env' import { refuseAgentSessionMutation } from './structured-agent-session-mutation-admission' +import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' import { retryPendingStructuredAgentSessionSettlement } from './structured-agent-session-settlement-retry' import { settleStaleSessionStateOnAcquire } from './structured-agent-session-stale-turn-verdict' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' @@ -34,166 +36,224 @@ import { type AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' -export function attachStructuredAgentSession( +export type StructuredAgentSessionAttachOptions = { + /** Provider-exit recovery: refuses once the ticket the restart was issued for is stale. */ + admitRecoveryTicket?: () => boolean + recordPhase?: AgentSessionCreatePhaseRecorder +} + +/** + * The attach itself, for a caller already inside the session's serialize. + * + * That is every caller that has to know what the session looks like RIGHT NOW: a hold, a send + * making sure it has an owner, provider-exit recovery. They run their + * check and this attach in one serialized step, so "the session has no child" is still true when + * the attach starts. `attachStructuredAgentSession` is this under `serialize`, for a client. + */ +export function attachStructuredAgentSessionUnderSerialize( context: StructuredAgentSessionAttachContext, callerKey: string, params: AgentSessionAttachParams, - admitRecoveryTicket?: () => boolean + options: StructuredAgentSessionAttachOptions = {} +): Promise> { + return context.tasks.trackAttach(runAttach(context, callerKey, params, options)) +} + +export function attachStructuredAgentSession( + context: StructuredAgentSessionAttachContext, + callerKey: string, + params: AgentSessionAttachParams ): Promise> { const sessionId = params.envelope.sessionId + // Tracked from enqueue, not from its turn on the queue: a quit drains a queued attach before it + // evicts, so no child is spawned behind the eviction and orphaned. const run = (recordPhase?: AgentSessionCreatePhaseRecorder) => - context.serialize(sessionId, async () => { - if (admitRecoveryTicket && !admitRecoveryTicket()) { - return refuseAgentSessionMutation({ - code: 'agent_session_checkpoint_stale', - message: 'The provider-exit recovery ticket is no longer current.' - }) - } - const unreconciled = await withAgentSessionCreatePhase('reconcile_leases', recordPhase, () => - context.reconcileLeases(sessionId) - ) - if (unreconciled) { - return refuseAgentSessionMutation(unreconciled) - } - await withAgentSessionCreatePhase('resolve_recovery', recordPhase, () => - context.runtimeState.resolveRecovery(sessionId) - ) - // Retries a durable provider-exit journal settlement before a new owner is reserved. Answers - // settled when the record has none pending, so every attach can ask unconditionally. - const settled = await withAgentSessionCreatePhase('settlement_retry', recordPhase, () => - retryPendingStructuredAgentSessionSettlement({ - deps: context.deps, - sessions: context.sessions, - sessionId, - params, - now: () => context.now() - }) - ) - if (!settled) { - return refuseAgentSessionMutation({ - code: 'agent_session_ownership_unknown', - message: 'The provider-exit terminal journal settlement is still pending; retry attach.' - }) - } - const eventSink = context.runtimeState.eventSinkFor(sessionId) - const probe = await withAgentSessionCreatePhase('probe_owner', recordPhase, () => - context.runtimeState.probeOwner(sessionId) - ) - const attached = await performAttach({ - store: context.deps.store, - adapter: context.deps.adapter, - journalRoot: context.deps.journalRoot, - eventSink: eventSink.sink, - onAcquiring: async () => { - const barrier = await eventSink.drained() - if (!barrier.ok) { - throw barrier.error + context.tasks.trackAttach( + context.serialize(sessionId, () => runAttach(context, callerKey, params, { recordPhase })) + ) + if (params.envelope.expectedRuntimeFence !== null) { + return run() + } + return withAgentSessionSpan(async (span) => { + const startedAtMs = Date.now() + const phases: Parameters[0][] = [] + try { + return await run((timing) => phases.push(timing)) + } finally { + addAgentSessionCreatePhaseAttributes(span, { + totalDurationMs: Math.max(0, Date.now() - startedAtMs), + phases + }) + } + }) +} + +async function runAttach( + context: StructuredAgentSessionAttachContext, + callerKey: string, + params: AgentSessionAttachParams, + options: StructuredAgentSessionAttachOptions +): Promise> { + const sessionId = params.envelope.sessionId + const recordPhase = options.recordPhase + if (options.admitRecoveryTicket && !options.admitRecoveryTicket()) { + return refuseAgentSessionMutation({ + code: 'agent_session_checkpoint_stale', + message: 'The provider-exit recovery ticket is no longer current.' + }) + } + const unreconciled = await withAgentSessionCreatePhase('reconcile_leases', recordPhase, () => + context.reconcileLeases(sessionId) + ) + if (unreconciled) { + return refuseAgentSessionMutation(unreconciled) + } + await withAgentSessionCreatePhase('resolve_recovery', recordPhase, () => + context.runtimeState.resolveRecovery(sessionId) + ) + // Retries a durable provider-exit journal settlement before a new owner is reserved. Answers + // settled when the record has none pending, so every attach can ask unconditionally. + const settled = await withAgentSessionCreatePhase('settlement_retry', recordPhase, () => + retryPendingStructuredAgentSessionSettlement({ + deps: context.deps, + sessions: context.sessions, + sessionId, + params, + now: () => context.now() + }) + ) + if (!settled) { + return refuseAgentSessionMutation({ + code: 'agent_session_ownership_unknown', + message: 'The provider-exit terminal journal settlement is still pending; retry attach.' + }) + } + const probe = await withAgentSessionCreatePhase('probe_owner', recordPhase, () => + context.runtimeState.probeOwner(sessionId) + ) + // A child this attach spawns writes through a sink this attempt owns. Only a successful + // attach makes it the session's; any other exit closes it with whatever the child queued. + const attemptSink = context.runtimeState.mintEventSink(sessionId) + let attemptSinkAdopted = false + // A lease handed back cleanly is what a resume replaces. A writer current as of that owner is + // rebased onto the fence this attach publishes, since the restart is the only thing that moved it. + const released = context.deps.store.getRecord(sessionId) + const resumedFromFence = + released && isResumableStructuredAgentSessionRecord(released) + ? released.lease.runtimeFence + : undefined + const attached = stampFailedCreateOwnerVerdict( + context.deps.store, + callerKey, + params.envelope, + await performAttach({ + store: context.deps.store, + adapter: context.deps.adapter, + journalRoot: context.deps.journalRoot, + eventSink: attemptSink.sink, + // The superseded child's writes settle into its own journal before a new child starts. + onAcquiring: async () => { + const barrier = await context.runtimeState.currentEventSink(sessionId)?.drained() + if (barrier && !barrier.ok) { + throw barrier.error + } + }, + authority: { + spawnToken: () => context.deps.mintSpawnToken?.() ?? randomUUID(), + claimKeyId: context.deps.claimKeyId, + handoffOperationId: params.envelope.clientOperationId, + probe, + ...(await pinnedAgentSessionLaunchArgs(context.deps.resolveLaunchArgs, params)), + ...(await pinnedAgentSessionLaunchEnv(context.deps.resolveLaunchEnv, params)) + }, + callerKey, + params, + now: () => context.now(), + recordPhase, + // Site 9: this closes the PRIOR map entry it drops, never the provisional + // journal — it has no reference to that one. `onAttached` owns that. + onAttachFailed: async () => { + await forgetStructuredAgentSession(context, sessionId) + context.runtimeState.currentEventSink(sessionId)?.close() + context.runtimeState.discardEventSink(sessionId) + }, + onAttached: async (attached, acquisitionGeneration, acquiredOwner, providerChildPhase) => { + const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? 0 + const previous = context.sessions.get(sessionId) + const previousFence = previous?.fence + // A re-attach to a live child keeps the sink that child already writes through. + const eventSink = acquiredOwner + ? attemptSink + : (context.runtimeState.currentEventSink(sessionId) ?? attemptSink) + // Site 8: the provisional journal has no owner until the map takes it, + // and the barrier below throws by design. + try { + if (acquiredOwner) { + // Before the drain: the buffered events are the new child's, never a stale row's. + await settleStaleSessionStateOnAcquire({ + journal: attached.journal, + sessionId, + fence, + acquisitionGeneration + }) } - eventSink.unbind() - }, - authority: { - spawnToken: () => context.deps.mintSpawnToken?.() ?? randomUUID(), - claimKeyId: context.deps.claimKeyId, - handoffOperationId: params.envelope.clientOperationId, - probe, - ...(await pinnedAgentSessionLaunchArgs(context.deps.resolveLaunchArgs, params)), - ...(await pinnedAgentSessionLaunchEnv(context.deps.resolveLaunchEnv, params)) - }, - callerKey, - params, - now: () => context.now(), - recordPhase, - // Site 9: this closes the PRIOR map entry it drops, never the provisional - // journal — it has no reference to that one. `onAttached` owns that. - onAttachFailed: async () => { - await forgetStructuredAgentSession(context, sessionId) - eventSink.close() - context.runtimeState.discardEventSink(sessionId) - }, - onAttached: async (attached, acquisitionGeneration, acquiredOwner) => { - const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? 0 - const previous = context.sessions.get(sessionId) - const previousFence = previous?.fence - // Site 8: the provisional journal has no owner until the map takes it, - // and the barrier below throws by design. + await bindAndDrain(eventSink, attached.journal, fence, (activity) => + context.subscribers.publish(sessionId, attached.journal, activity) + ) + } catch (error) { + await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) + throw error + } + // Site 10: a `set` over a live entry would orphan its handle — and a + // close that REJECTED did not release it. The replacement is therefore + // ABORTED rather than completed over a handle nothing can reach again: + // `previous` stays indexed, so teardown still owns it and can retry. + if (previous && previous.journal !== attached.journal) { try { - if (acquiredOwner) { - // Before the drain: the buffered events are the new child's, never a stale row's. - await settleStaleSessionStateOnAcquire({ - journal: attached.journal, - sessionId, - fence, - acquisitionGeneration - }) - } - await bindAndDrain(eventSink, attached.journal, fence, (activity) => - context.subscribers.publish(sessionId, attached.journal, activity) - ) + await previous.journal.close() } catch (error) { await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) throw error } - // Site 10: a `set` over a live entry would orphan its handle — and a - // close that REJECTED did not release it. The replacement is therefore - // ABORTED rather than completed over a handle nothing can reach again: - // `previous` stays indexed, so teardown still owns it and can retry. - if (previous && previous.journal !== attached.journal) { - try { - await previous.journal.close() - } catch (error) { - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error - } - } - context.sessions.set(sessionId, { - journal: attached.journal, - params, - fence, - hasProviderChild: true, - acquisitionGeneration: acquisitionGeneration ?? previous?.acquisitionGeneration ?? null - }) - await recoverStructuredRewind( - context.deps.store, - sessionId, - attached.journal, - fence, - context.deps.adapter, - context.now - ) - await recoverInterruptedCompaction(context.deps.store, sessionId, attached.journal, fence) - if (attached.recovery) { - context.subscribers.reset(sessionId, attached.journal, attached.recovery.reset, fence) - } else if (previousFence !== undefined && previousFence !== fence) { - context.subscribers.snapshot(sessionId, attached.journal, fence) - } else { - context.subscribers.publish(sessionId, attached.journal) - } } - }) - // Why: a failed attach that left no session behind must not strand a bound sink; the runtime - // caches one per session id and would hand this same closed instance to the next attempt. - if (!attached.ok && !context.sessions.has(sessionId)) { - eventSink.close() - context.runtimeState.discardEventSink(sessionId) + context.runtimeState.adoptEventSink(sessionId, eventSink) + attemptSinkAdopted = eventSink === attemptSink + context.sessions.set(sessionId, { + journal: attached.journal, + params, + fence, + hasProviderChild: true, + // A re-attach to a live child keeps what that child already proved. + providerChildPhase: acquiredOwner + ? providerChildPhase + : (previous?.providerChildPhase ?? 'ready'), + acquisitionGeneration: acquisitionGeneration ?? previous?.acquisitionGeneration ?? null, + resumedFromFence: acquiredOwner ? resumedFromFence : previous?.resumedFromFence + }) + await recoverStructuredRewind( + context.deps.store, + sessionId, + attached.journal, + fence, + context.deps.adapter, + context.now + ) + await recoverInterruptedCompaction(context.deps.store, sessionId, attached.journal, fence) + if (attached.recovery) { + context.subscribers.reset(sessionId, attached.journal, attached.recovery.reset, fence) + } else if (previousFence !== undefined && previousFence !== fence) { + context.subscribers.snapshot(sessionId, attached.journal, fence) + } else { + context.subscribers.publish(sessionId, attached.journal) + } + } + }).finally(() => { + if (!attemptSinkAdopted) { + attemptSink.close() } - return attached }) - const attaching = - params.envelope.expectedRuntimeFence === null - ? withAgentSessionSpan(async (span) => { - const startedAtMs = Date.now() - const phases: Parameters[0][] = [] - try { - return await run((timing) => phases.push(timing)) - } finally { - addAgentSessionCreatePhaseAttributes(span, { - totalDurationMs: Math.max(0, Date.now() - startedAtMs), - phases - }) - } - }) - : run() - return context.tasks.trackAttach(attaching) + ) + return attached } /** Binds the sink to the journal and waits for the barrier the host publishes diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index a7ca961d13ff..d83ca49a79c5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -106,6 +106,7 @@ describe('Claude root-exit eviction', () => { params, fence, hasProviderChild: true, + providerChildPhase: 'ready', acquisitionGeneration: acquisition.acquisitionGeneration ?? null } ] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index c2702fcd66df..7681e0f34362 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -26,7 +26,8 @@ export class StructuredAgentSessionClientDelivery { constructor( private readonly sessions: Map, now: () => number, - deps: () => StructuredAgentSessionHostDeps + deps: () => StructuredAgentSessionHostDeps, + private readonly onJournalActivity?: (sessionId: string) => void ) { this.statusFeed = createStructuredAgentSessionHostStatusFeed({ sessions, now, deps }) this.turnCompletionFeed = new StructuredAgentSessionTurnCompletionFeed({ sessions, now }) @@ -78,6 +79,7 @@ export class StructuredAgentSessionClientDelivery { // Derived here rather than per-subscriber: this edge runs whether or not anyone is // subscribed, which is the whole reason a backgrounded chat can complete at all. this.turnCompletionFeed.observe(sessionId, journal) + this.onJournalActivity?.(sessionId) } private requireJournal(sessionId: string): AgentSessionJournal { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts index 1141f8211748..b9e8f8e506d7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts @@ -80,6 +80,7 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe params: {} as StructuredAgentSessionHostSession['params'], fence: 1, hasProviderChild: false, + providerChildPhase: 'ready', acquisitionGeneration: null } } @@ -115,11 +116,15 @@ function attachContext( bind: () => undefined, close: () => undefined } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a partial context double; the attach reads only the members defined here. return { deps: { store: { getRecord: () => null }, claimKeyId: 'key-1', journalRoot: root }, runtimeState: { resolveRecovery: async () => undefined, eventSinkFor: () => eventSink, + currentEventSink: () => eventSink, + mintEventSink: () => eventSink, + adoptEventSink: () => undefined, probeOwner: async () => ({ outcome: 'pid-absent' }), discardEventSink: () => undefined }, @@ -215,8 +220,10 @@ describe('the attach orchestration', () => { bind: () => undefined, close: () => undefined } - context.runtimeState.eventSinkFor = (() => - failing) as unknown as typeof context.runtimeState.eventSinkFor + // The re-attach binds the sink the live child already writes through. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a partial sink double; the attach reads only drained/bind/close from it. + context.runtimeState.currentEventSink = (() => + failing) as unknown as typeof context.runtimeState.currentEventSink await expect(attachStructuredAgentSession(context, 'caller-1', attachParams)).rejects.toThrow( 'sink barrier failed' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts index 64b7b9d03b5e..d553fbaecf37 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import { dispatchRejectionReasonIsInternal } from '../../../shared/structured-agent-session-dispatch-rejection' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { captureUnfinishedStructuredAgentSessionWork, @@ -185,7 +186,11 @@ describe('dead structured-session generation settlement', () => { const settledSnapshot = journal.snapshot() const closedJournal: Pick< AgentSessionJournal, - 'snapshot' | 'submissions' | 'markPendingSubmissionsUnknown' | 'appendLifecycleBatch' + | 'snapshot' + | 'submissions' + | 'markPendingSubmissionsUnknown' + | 'rejectPendingSubmissions' + | 'appendLifecycleBatch' > = { snapshot: () => ({ ...settledSnapshot, @@ -195,6 +200,9 @@ describe('dead structured-session generation settlement', () => { markPendingSubmissionsUnknown: async () => { throw new Error('journal_closed') }, + rejectPendingSubmissions: async () => { + throw new Error('journal_closed') + }, appendLifecycleBatch: async () => { throw new Error('journal_closed') } @@ -249,6 +257,33 @@ describe('dead structured-session generation settlement', () => { ]) }) + it('rejects a send a child that never started left pending with its diagnostic, in words', async () => { + await journal.appendSubmission({ + clientMessageId: 'client-held', + payloadFingerprint: 'fingerprint', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hello?' }] }, + fence: 7 + }) + + await settleStructuredAgentSessionDeadGeneration({ + journal, + sessionId: SESSION, + fence: 7, + settlementId: `provider-exit:${SESSION}:7:generation-1`, + pendingSubmissionReason: 'provider_closed_before_acknowledgement', + verdict: { state: 'interrupted', completedAt: 1_000 }, + unexpectedExitReason: 'claude stream-json exited (code 1): not signed in', + exitedDuringStartup: true + }) + + const reason = + 'The provider stopped before it finished starting: claude stream-json exited (code 1): not signed in.' + expect(journal.submissions()).toEqual([ + expect.objectContaining({ clientMessageId: 'client-held', dispatchState: 'rejected', reason }) + ]) + expect(dispatchRejectionReasonIsInternal(reason)).toBe(false) + }) + it("keeps a subagent's settled rows the subagent's, in one batch and after a reopen", async () => { // One batch settles rows several agents wrote and names none of them. Each // row keeps the producer its first write named, including after a replay. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 9a3f3a7c091b..1b5f73fdbdae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -26,15 +26,50 @@ export const MAX_UNEXPECTED_EXIT_REASON_CHARS = 512 /** The cause is the only thing separating an auth failure from an OOM kill, so it is carried * into the copy rather than left in the durable record nothing renders. */ export function unexpectedProviderExitOutcome(reason?: string): string { - const detail = reason - ?.slice(0, MAX_UNEXPECTED_EXIT_REASON_CHARS) - .trim() - .replace(/[.\s]+$/, '') + const detail = exitReasonDetail(reason) return detail ? `The provider stopped while this response was in progress: ${detail}. You can continue in this conversation.` : UNEXPECTED_PROVIDER_EXIT_OUTCOME } +/** A restart that produced no child, answered to the send that asked for it; its cause is the + * whole story, and nothing is remembered, so the next try is a fresh one. A new chat is offered + * only when the host holds nothing this chat could restart from. */ +export function ownerRestartFailedOutcome(input: { + agentName: string + reason?: string + resumable: boolean +}): string { + const detail = exitReasonDetail(input.reason) + const failed = detail + ? `${input.agentName} couldn't restart: ${detail}.` + : `${input.agentName} couldn't restart.` + return input.resumable ? failed : `${failed} Start a new chat to continue.` +} + +/** A start that never finished has no response to interrupt; its cause is the whole story. */ +export function providerStartupFailureOutcome(reason?: string): string { + const detail = exitReasonDetail(reason) + return detail + ? `The provider stopped before it finished starting: ${detail}.` + : 'The provider stopped before it finished starting.' +} + +/** Why a send a child that never started left unwritten was rejected. The child's own diagnostic is + * the cause the user can act on, so it is the reason, in the words the chat row uses. */ +export function providerStartupFailureRejection(cause?: unknown): string { + return providerStartupFailureOutcome( + cause === undefined ? undefined : cause instanceof Error ? cause.message : String(cause) + ) +} + +function exitReasonDetail(reason: string | undefined): string | undefined { + return reason + ?.slice(0, MAX_UNEXPECTED_EXIT_REASON_CHARS) + .trim() + .replace(/[.\s]+$/, '') +} + type DeadGenerationSubmission = Pick< ReturnType[number], 'clientMessageId' | 'dispatchState' | 'recovered' @@ -43,6 +78,7 @@ type DeadGenerationSubmission = Pick< export type DeadGenerationJournal = { appendLifecycleBatch: AgentSessionJournal['appendLifecycleBatch'] markPendingSubmissionsUnknown: AgentSessionJournal['markPendingSubmissionsUnknown'] + rejectPendingSubmissions: AgentSessionJournal['rejectPendingSubmissions'] snapshot: () => Pick, 'items'> pendingSubmissions?: AgentSessionJournal['pendingSubmissions'] submissions?: () => DeadGenerationSubmission[] @@ -106,6 +142,8 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { showUnexpectedExitOutcome?: boolean /** Why the provider stopped, when the host has it. Rendered with the outcome copy. */ unexpectedExitReason?: string + /** The provider never finished starting; the outcome says so instead of naming a response. */ + exitedDuringStartup?: boolean onError?: (sessionId: string, error: unknown) => void }): Promise { try { @@ -114,7 +152,15 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { return true } - await input.journal.markPendingSubmissionsUnknown(input.fence, input.pendingSubmissionReason) + // A child that never proved its start accepted nothing — input is written only after it + // initializes — so every send it left unanswered is provably unwritten and is rejected with the + // child's own diagnostic. A proven child's unanswered sends stay in doubt. + await (input.exitedDuringStartup + ? input.journal.rejectPendingSubmissions( + input.fence, + providerStartupFailureRejection(input.unexpectedExitReason) + ) + : input.journal.markPendingSubmissionsUnknown(input.fence, input.pendingSubmissionReason)) const items = input.journal.snapshot().items const mutations: JournalLifecycleMutationInput[] = [] if (showUnexpectedExitOutcome) { @@ -123,7 +169,11 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { identity: { provider: 'orca', clientMessageId: input.settlementId }, body: { kind: 'status', - text: boundJournalStatusText(unexpectedProviderExitOutcome(input.unexpectedExitReason)) + text: boundJournalStatusText( + input.exitedDuringStartup + ? providerStartupFailureOutcome(input.unexpectedExitReason) + : unexpectedProviderExitOutcome(input.unexpectedExitReason) + ) } }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 728942b63475..89a492130884 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -1,12 +1,11 @@ -import { attachStructuredAgentSession } from './structured-agent-session-attach-orchestration' -import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionLifecycleEvent } from './structured-agent-session-adapter' import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' -import { resumeHeldStructuredAgentSession } from './structured-agent-session-hold-resume' +import type { StructuredAgentSessionHolds } from './structured-agent-session-holds' +import { settleStructuredAgentSessionProviderStarted } from './structured-agent-session-provider-started' import { isStructuredAgentSessionRecoveryTicketCurrent, settleUnexpectedStructuredAgentSessionExit @@ -24,9 +23,11 @@ export class StructuredAgentSessionEventRecovery { publishFence: (sessionId: string, session: StructuredAgentSessionHostSession) => void publishStatus?: (sessionId: string) => void hasResumeCapableHolder: (sessionId: string) => boolean + restartReleaseGrace: (sessionId: string) => void serialize: (sessionId: string, task: () => Promise) => Promise now: () => number - attachContext: () => StructuredAgentSessionAttachContext + /** The one restart every asker shares; the holds put an unheld child on the idle clock. */ + ensureProviderChild: StructuredAgentSessionHolds['ensureProviderChild'] onBarrierError: (sessionId: string, error: unknown) => void } ) {} @@ -65,23 +66,29 @@ export class StructuredAgentSessionEventRecovery { } async handle(event: StructuredAgentSessionLifecycleEvent): Promise { + if (event.type === 'started') { + return settleStructuredAgentSessionProviderStarted(this.context, event) + } const ticket = await settleUnexpectedStructuredAgentSessionExit(this.context, event) if (!ticket) { return } + // One serialized step with the ticket check inside it: a hold or a send that got there first + // has already replaced the owner, and this step finds that child and attaches nothing — or, + // once the lease has moved on, refuses on the stale ticket rather than spawning a second child. try { - await resumeHeldStructuredAgentSession({ - sessionId: ticket.sessionId, - deps: this.context.deps, - now: this.context.now, - attach: (params) => - attachStructuredAgentSession( - this.context.attachContext(), - 'trusted-local:provider-exit-recovery', - params, - () => isStructuredAgentSessionRecoveryTicketCurrent(this.context, ticket) - ) - }) + const resumed = await this.context.serialize(ticket.sessionId, () => + this.context.ensureProviderChild(ticket.sessionId, { + admitRecoveryTicket: () => + isStructuredAgentSessionRecoveryTicketCurrent(this.context, ticket) + }) + ) + if (!resumed.ok && isStructuredAgentSessionRecoveryTicketCurrent(this.context, ticket)) { + this.context.onBarrierError( + ticket.sessionId, + new Error(`${resumed.refusal.code}: ${resumed.refusal.message}`) + ) + } } catch (error) { if (isStructuredAgentSessionRecoveryTicketCurrent(this.context, ticket)) { this.context.onBarrierError(ticket.sessionId, error) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-owner-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-owner-verdict.test.ts new file mode 100644 index 000000000000..3d064eed7e0b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-owner-verdict.test.ts @@ -0,0 +1,137 @@ +// A durably failed create tells a client what the host proved about the provider process, so a +// client can tell "retry under a new operation" (exited) from "the session may exist" (anything else). + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + AgentSessionAcquisitionExitUnprovenError, + AgentSessionAcquisitionRefusal, + AgentSessionAcquisitionRootExitObservedError, + type StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +const EXIT_REASON = 'claude stream-json exited (code 1): stderr tail' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-failed-create-verdict-')) + resetHostTestOperationIds() + acquire = vi.fn(async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + origin: 'created', + mintedAtFence: fence, + observedAt: NOW + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + releaseAcquisition: vi.fn(async () => true), + dispatch: vi.fn(async () => ({ state: 'admitted' as const })), + cancelTurn: vi.fn(async () => ({ cancelled: true })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined) + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => NOW + }) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +describe('failed create owner verdict', () => { + it('answers an exit-proven failure as exited on the first call and its replay, and a new operation starts fresh', async () => { + // The cleanup's release proves the whole tree gone: the common failed start. + acquire.mockRejectedValueOnce(new Error(EXIT_REASON)) + const first = hostTestAttachParams(null) + const refusal = { + code: 'agent_session_operation_invalid', + message: EXIT_REASON, + ownerVerdict: 'exited' + } + + await expect(host.attach(CALLER, first)).resolves.toEqual({ ok: false, refusal }) + await expect(host.attach(CALLER, first)).resolves.toEqual({ ok: false, refusal }) + expect(acquire).toHaveBeenCalledOnce() + + const retry = hostTestAttachParams(null) + expect(retry.envelope.clientOperationId).not.toBe(first.envelope.clientOperationId) + await expect(host.attach(CALLER, retry)).resolves.toMatchObject({ ok: true }) + expect(acquire).toHaveBeenCalledTimes(2) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + }) + + it('answers a first-hand root exit as exited on the first call, in the shape its replay takes', async () => { + acquire.mockRejectedValueOnce( + new AgentSessionAcquisitionRootExitObservedError(new Error(EXIT_REASON)) + ) + const first = hostTestAttachParams(null) + const refusal = { + code: 'agent_session_operation_invalid', + message: EXIT_REASON, + ownerVerdict: 'exited' + } + + await expect(host.attach(CALLER, first)).resolves.toEqual({ ok: false, refusal }) + await expect(host.attach(CALLER, first)).resolves.toEqual({ ok: false, refusal }) + expect(acquire).toHaveBeenCalledOnce() + + await expect(host.attach(CALLER, hostTestAttachParams(null))).resolves.toMatchObject({ + ok: true + }) + expect(acquire).toHaveBeenCalledTimes(2) + }) + + it('answers an acquisition refusal with its verdict directly', async () => { + acquire.mockRejectedValueOnce(new AgentSessionAcquisitionRefusal('not signed in')) + + await expect(host.attach(CALLER, hostTestAttachParams(null))).resolves.toEqual({ + ok: false, + refusal: { + code: 'agent_session_operation_invalid', + message: 'not signed in', + ownerVerdict: 'exited' + } + }) + }) + + it('never claims exited when the failed attempt could not prove its process gone', async () => { + acquire.mockRejectedValueOnce(new AgentSessionAcquisitionExitUnprovenError(new Error('hung'))) + const first = hostTestAttachParams(null) + + await expect(host.attach(CALLER, first)).rejects.toThrow() + const replay = await host.attach(CALLER, first) + + expect(replay).toMatchObject({ + ok: false, + refusal: { code: 'agent_session_ownership_unknown', ownerVerdict: 'unverifiable' } + }) + expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-refusal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-refusal.ts new file mode 100644 index 000000000000..a5e00fde53c0 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-refusal.ts @@ -0,0 +1,96 @@ +import type { + AgentSessionAttachResult, + AgentSessionMutationResult, + AgentSessionWireRefusal +} from '../../../shared/agent-session-wire' +import type { + AgentSessionOperationOutcome, + AgentSessionOperationRow +} from '../../../shared/agent-session-operation-ledger' +import { agentSessionLeaseOwnerVerdict } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionAcquisitionExitProof } from '../../runtime/agent-session-acquisition-failure-settlement' +import { + AgentSessionAcquisitionExitProvenError, + AgentSessionAcquisitionExitUnprovenError, + AgentSessionAcquisitionRefusal, + AgentSessionAcquisitionRootExitObservedError, + isAgentSessionPreSpawnError +} from './structured-agent-session-adapter' + +/** What a failed acquisition proved about its process, and the outcome its operation settles to. */ +export function failedAcquisitionSettlement(error: unknown): { + exitProof: AgentSessionAcquisitionExitProof + outcome: Extract +} { + if (error instanceof AgentSessionAcquisitionExitUnprovenError) { + const outcome = { code: 'agent_session_ownership_unknown', message: error.message } + return { exitProof: 'unproven', outcome: { status: 'failed', ...outcome } } + } + const exitProof = isAgentSessionPreSpawnError(error) + ? 'processless' + : error instanceof AgentSessionAcquisitionRootExitObservedError + ? 'root-exit-observed' + : 'exit-proven' + const message = error instanceof Error ? error.message : String(error) + const code = + error instanceof AgentSessionAcquisitionRefusal ? error.code : 'agent_session_operation_invalid' + return { exitProof, outcome: { status: 'failed', code, message } } +} + +/** A failed acquisition answered as a refusal on the first call, in the shape its replay takes; + * null leaves the error to the store-failure classification. */ +export function failedAcquisitionRefusal( + error: unknown +): { ok: false; refusal: AgentSessionWireRefusal } | null { + if (error instanceof AgentSessionAcquisitionRefusal) { + return { ok: false, refusal: { code: error.code, message: error.message } } + } + // A proven exit is a settled fact; its message is the provider's own diagnostic. + if ( + error instanceof AgentSessionAcquisitionRootExitObservedError || + error instanceof AgentSessionAcquisitionExitProvenError + ) { + return { + ok: false, + refusal: { code: 'agent_session_operation_invalid', message: error.message } + } + } + return null +} + +/** Only a durably failed operation says anything about retrying under a new one. */ +export function failedCreateRefusal( + refusal: AgentSessionWireRefusal, + status: AgentSessionOperationOutcome['status'], + record: AgentSessionRecord | null +): { ok: false; refusal: AgentSessionWireRefusal } { + return status === 'failed' && record + ? { + ok: false, + refusal: { ...refusal, ownerVerdict: agentSessionLeaseOwnerVerdict(record.lease) } + } + : { ok: false, refusal } +} + +/** The one place a create refusal learns its verdict: from the durable row this operation + * settled to, so every refusal shape answers the same fact and no site can forget the stamp. */ +export function stampFailedCreateOwnerVerdict( + store: { + getOperationRow: (callerKey: string, operationId: string) => AgentSessionOperationRow | null + getRecord: (sessionId: string) => AgentSessionRecord | null + }, + callerKey: string, + envelope: { sessionId: string; clientOperationId: string }, + result: AgentSessionMutationResult +): AgentSessionMutationResult { + if (result.ok) { + return result + } + const row = store.getOperationRow(callerKey, envelope.clientOperationId) + return failedCreateRefusal( + result.refusal, + row?.outcome.status ?? 'pending', + store.getRecord(envelope.sessionId) + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-sink-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-sink-release.test.ts new file mode 100644 index 000000000000..c61d93cbb8fd --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-sink-release.test.ts @@ -0,0 +1,137 @@ +// A child that dies between spawn and journal attach can still write through the host's event +// sink, which attach unbound and never re-bound. That queue must die with the failed create, or +// the next attach's drain barrier and shutdown's flush wait on it forever. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + AgentSessionPreSpawnError, + type StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +const EXIT_REASON = 'claude stream-json exited (code 1): claude: not signed in' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-failed-create-sink-')) + resetHostTestOperationIds() + acquire = vi.fn(async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + // A resume continues the chain the first start created. + origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', + mintedAtFence: fence, + observedAt: NOW + }, + acquisitionGeneration: `generation-${fence}` + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + releaseAcquisition: vi.fn(async () => true), + dispatch: vi.fn(async () => ({ state: 'admitted' as const })), + cancelTurn: vi.fn(async () => ({ cancelled: true })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined) + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => NOW + }) +}) + +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) + +describe('a create that fails after its child wrote through the unbound sink', () => { + it.each([ + // The common failed start: answered as a refusal. + ['refused', new Error(EXIT_REASON)], + // A failure the attach cannot classify still throws, and must release the sink too. + ['thrown', new AgentSessionPreSpawnError(new Error(EXIT_REASON))] + ])( + 'releases the sink when %s, so a new create and shutdown both proceed', + async (_how, cause) => { + acquire.mockImplementationOnce(async ({ events }) => { + // The published child's exit reached the translator before any journal was attached. + events?.setActivity?.(null) + throw cause + }) + + const failed = host.attach(CALLER, hostTestAttachParams(null)) + await (cause instanceof AgentSessionPreSpawnError + ? expect(failed).rejects.toThrow(EXIT_REASON) + : expect(failed).resolves.toMatchObject({ ok: false, refusal: { message: EXIT_REASON } })) + + await expect(host.attach(CALLER, hostTestAttachParams(null))).resolves.toMatchObject({ + ok: true + }) + await expect(host.flushAllStreamedEvents()).resolves.toBeUndefined() + expect(acquire).toHaveBeenCalledTimes(2) + } + ) + + it.each([ + ['refused', new Error(EXIT_REASON)], + ['thrown', new AgentSessionPreSpawnError(new Error(EXIT_REASON))] + ])('releases the sink when a resume of a still-indexed session is %s', async (_how, cause) => { + await expect(host.attach(CALLER, hostTestAttachParams(null))).resolves.toMatchObject({ + ok: true + }) + const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + await host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'provider exited', + cause: 'unexpected-exit', + fence: exitedFence, + acquisitionGeneration: `generation-${exitedFence}` + }) + const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + acquire.mockImplementationOnce(async ({ events }) => { + events?.setActivity?.(null) + throw cause + }) + + // The session stays indexed across this failure: it is a resume, not a create. + const failed = host.attach(CALLER, hostTestAttachParams(releasedFence)) + await (cause instanceof AgentSessionPreSpawnError + ? expect(failed).rejects.toThrow(EXIT_REASON) + : expect(failed).resolves.toMatchObject({ ok: false })) + + const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + await expect(host.attach(CALLER, hostTestAttachParams(fence))).resolves.toMatchObject({ + ok: true + }) + expect(acquire).toHaveBeenCalledTimes(3) + // Only the adopted child's sink still takes writes: the failed attempt's closed with it, and + // the exited generation's closed when the resume replaced it. + const [exited, failedAttempt, resumed] = acquire.mock.calls.map(([input]) => input.events) + expect(failedAttempt?.tryPublish?.()).toEqual({ accepted: false, reason: 'closed' }) + expect(exited?.tryPublish?.()).toEqual({ accepted: false, reason: 'closed' }) + expect(resumed?.tryPublish?.()).toEqual({ accepted: true }) + await expect(host.flushAllStreamedEvents()).resolves.toBeUndefined() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts index 2ff6697eae95..20531bd64653 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts @@ -138,6 +138,7 @@ async function workingSession(): Promise<{ }, fence: 1, hasProviderChild: true, + providerChildPhase: 'ready', acquisitionGeneration: null } ] @@ -171,11 +172,15 @@ function attachContext( bind: () => undefined, close: () => undefined } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a partial context double; the attach reads only the members defined here. return { deps: { store: { getRecord: () => null }, claimKeyId: 'key-1', journalRoot: root }, runtimeState: { resolveRecovery: async () => undefined, eventSinkFor: () => eventSink, + currentEventSink: () => eventSink, + mintEventSink: () => eventSink, + adoptEventSink: () => undefined, probeOwner: async () => ({ outcome: 'pid-absent' }), discardEventSink: () => undefined }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts index bd3a840d9ad9..974a69b1c3c8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts @@ -1,9 +1,18 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { runKeyedSerializedOperation } from '../../cli/keyed-promise-queue' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' import { StructuredAgentSessionHolds } from './structured-agent-session-holds' const GRACE_MS = 15_000 const pendingHolds: StructuredAgentSessionHolds[] = [] +/** The host's per-session queue: a second hold waits for the attach the first one is running. */ +function keyedSerialize() { + const chains = new Map>() + return (sessionId: string, task: () => Promise) => + runKeyedSerializedOperation(chains, sessionId, task) +} + function resumeHarness() { const resumeGate = Promise.withResolvers() let child = false @@ -11,19 +20,23 @@ function resumeHarness() { const evict = vi.fn(async () => { child = false }) + const resume = vi.fn(async () => { + await resumeGate.promise + child = true + return { ok: true as const } + }) const holds = new StructuredAgentSessionHolds({ - resume: async () => { - await resumeGate.promise - child = true - }, + resume, + serialize: keyedSerialize(), hasProviderChild: () => child, - isTurnActive: () => turnActive, + hasOwedWork: () => turnActive, evict, graceMs: GRACE_MS }) pendingHolds.push(holds) return { holds, + resume, resumeGate, evict, hasChild: () => child, @@ -165,125 +178,83 @@ describe('a surface leaving while its structured session resumes', () => { expect(evict).toHaveBeenCalledExactlyOnceWith('session-1') }) - it.each([false, true])( - 'keeps a reused holder when old resume fails (replacement finished=%s)', - async (replacementFinished) => { - const firstGate = Promise.withResolvers() - const replacementGate = Promise.withResolvers() - let child = false - const resume = vi - .fn() - .mockImplementationOnce(() => firstGate.promise) - .mockImplementationOnce(async () => { - await replacementGate.promise - child = true - }) - const evict = vi.fn(async () => {}) - const holds = new StructuredAgentSessionHolds({ - resume, - hasProviderChild: () => child, - isTurnActive: () => false, - evict, - graceMs: GRACE_MS - }) - pendingHolds.push(holds) - const first = holds.hold('session-1', 'same-holder') - const rejected = expect(first).rejects.toThrow('old acquisition failed') - holds.release('session-1', 'same-holder') - const replacement = holds.hold('session-1', 'same-holder') - if (replacementFinished) { - replacementGate.resolve() - await replacement - } - - firstGate.reject(new Error('old acquisition failed')) - await rejected - expect(holds.isHeld('session-1')).toBe(true) - replacementGate.resolve() - await replacement - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(evict).not.toHaveBeenCalled() - - holds.release('session-1', 'same-holder') - await vi.advanceTimersByTimeAsync(GRACE_MS) - expect(evict).toHaveBeenCalledExactlyOnceWith('session-1') - } - ) + it('lets a holder that left and came back make its own attempt behind a failing one, and its failure releases it', async () => { + const { holds, resume, resumeGate, evict } = resumeHarness() + const first = holds.hold('session-1', 'same-holder') + const firstRejected = expect(first).rejects.toThrow('acquisition failed') + holds.release('session-1', 'same-holder') + const replacement = holds.hold('session-1', 'same-holder') + const replacementRejected = expect(replacement).rejects.toThrow('acquisition failed') + await vi.advanceTimersByTimeAsync(0) + expect(holds.isHeld('session-1')).toBe(true) + expect(resume).toHaveBeenCalledOnce() - it('removes a failed replacement while the released old hold is still pending', async () => { - const firstGate = Promise.withResolvers() - const resume = vi - .fn() - .mockImplementationOnce(() => firstGate.promise) - .mockRejectedValueOnce(new Error('replacement acquisition failed')) + // The gate stays rejected, so the replacement's own attempt fails the same way. + resumeGate.reject(new Error('acquisition failed')) + await Promise.all([firstRejected, replacementRejected]) + + expect(resume).toHaveBeenCalledTimes(2) + expect(holds.isHeld('session-1')).toBe(false) + expect(holds.isReleasePending('session-1')).toBe(false) + await vi.advanceTimersByTimeAsync(GRACE_MS * 2) + expect(evict).not.toHaveBeenCalled() + }) + + it('keeps a re-hold that finds the child the failing attempt ahead of it left behind', async () => { + const gate = Promise.withResolvers() + let child = false const holds = new StructuredAgentSessionHolds({ - resume, - hasProviderChild: () => false, - isTurnActive: () => false, + // The child is up before the attempt settles, and then the attempt fails behind it. + resume: async () => { + child = true + await gate.promise + return { ok: true as const } + }, + serialize: keyedSerialize(), + hasProviderChild: () => child, + hasOwedWork: () => false, evict: async () => {}, graceMs: GRACE_MS }) pendingHolds.push(holds) const first = holds.hold('session-1', 'same-holder') - const rejected = expect(first).rejects.toThrow('old acquisition failed') + const rejected = expect(first).rejects.toThrow('acquisition failed') holds.release('session-1', 'same-holder') + const replacement = holds.hold('session-1', 'same-holder') - await expect(holds.hold('session-1', 'same-holder')).rejects.toThrow( - 'replacement acquisition failed' - ) - expect(holds.isHeld('session-1')).toBe(false) - expect(holds.isReleasePending('session-1')).toBe(false) - - firstGate.reject(new Error('old acquisition failed')) + gate.reject(new Error('acquisition failed')) await rejected - expect(holds.isHeld('session-1')).toBe(false) + // The first hold's failure released only the holder it added, at the incarnation it added; + // the replacement then ran, found the child, and kept the holder it re-took. + await replacement + + expect(holds.isHeld('session-1')).toBe(true) + expect(holds.isReleasePending('session-1')).toBe(false) }) - it.each(['old-holder', 'different-holder'])( - 'releases the old acquisition after replacement %s fails, once its turn finishes', - async (replacementHolder) => { - const firstGate = Promise.withResolvers() - const replacementGate = Promise.withResolvers() - let child = false - let turnActive = true - const resume = vi - .fn() - .mockImplementationOnce(async () => { - await firstGate.promise - child = true - }) - .mockImplementationOnce(() => replacementGate.promise) - const evict = vi.fn(async () => { - child = false - }) - const holds = new StructuredAgentSessionHolds({ - resume, - hasProviderChild: () => child, - isTurnActive: () => turnActive, - evict, - graceMs: GRACE_MS + it('starts a fresh resume once the failed one has settled', async () => { + let child = false + const resume = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error('first acquisition failed')) + .mockImplementationOnce(async () => { + child = true + return { ok: true } }) - pendingHolds.push(holds) - const first = holds.hold('session-1', 'old-holder') - holds.release('session-1', 'old-holder') - const replacement = holds.hold('session-1', replacementHolder) - const rejected = expect(replacement).rejects.toThrow('replacement acquisition failed') - firstGate.resolve() - await first - expect(holds.isReleasePending('session-1')).toBe(false) - - replacementGate.reject(new Error('replacement acquisition failed')) - await rejected - expect(holds.isHeld('session-1')).toBe(false) - expect(holds.isReleasePending('session-1')).toBe(true) - await vi.advanceTimersByTimeAsync(GRACE_MS) - expect(evict).not.toHaveBeenCalled() - expect(child).toBe(true) - - turnActive = false - await vi.advanceTimersByTimeAsync(GRACE_MS) - expect(evict).toHaveBeenCalledExactlyOnceWith('session-1') - expect(child).toBe(false) - } - ) + const holds = new StructuredAgentSessionHolds({ + resume, + serialize: keyedSerialize(), + hasProviderChild: () => child, + hasOwedWork: () => false, + evict: async () => {}, + graceMs: GRACE_MS + }) + pendingHolds.push(holds) + + await expect(holds.hold('session-1', 'chat-1')).rejects.toThrow('first acquisition failed') + await holds.hold('session-1', 'chat-1') + + expect(resume).toHaveBeenCalledTimes(2) + expect(holds.isHeld('session-1')).toBe(true) + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume.ts index 5ebdfed01145..9ea59e18487c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume.ts @@ -1,52 +1,130 @@ -// Giving a held session its provider child back. +// Giving a session its provider child back. // // This is the replacement for the startup resume, and the difference is only in WHO asks: the same -// eligibility rule, run when a surface binds instead of when the app launches. A write-capable hold -// must fail when acquisition is refused so the surface never mistakes a readable journal for a live -// provider child. +// eligibility rule, run when a surface binds, when a send finds the owner gone, or when a child +// exits under an open surface — never when the app launches. It runs inside the session's +// serialize, with the attach it is given, so the eligibility it reads is the one the attach acts +// on. A write-capable hold must fail when acquisition is refused so the surface never mistakes a +// readable journal for a live provider child. import type { AgentSessionAttachResult, - AgentSessionMutationResult + AgentSessionMutationResult, + AgentSessionWireRefusal } from '../../../shared/agent-session-wire' -import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { isAgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals' +import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' +import { + attachStructuredAgentSessionUnderSerialize, + type StructuredAgentSessionAttachOptions +} from './structured-agent-session-attach-orchestration' +import { failedCreateRefusal } from './structured-agent-session-failed-create-refusal' import { adapterSupportsRecord } from './structured-agent-session-provider-support' import { structuredAgentSessionResumeOperationId, structuredAgentSessionResumeParams } from './structured-agent-session-resume-eligibility' +/** A resume answers with the attach's own refusal, verdict and all, so the asker can tell a lease + * someone else is settling from an owner that will not come back. */ +export type StructuredAgentSessionResumeOutcome = + | { ok: true } + | { ok: false; refusal: AgentSessionWireRefusal } + export async function resumeHeldStructuredAgentSession(input: { sessionId: string - deps: StructuredAgentSessionHostDeps - now: () => number - attach: ( - params: AgentSessionAttachParams - ) => Promise> -}): Promise { - const record = input.deps.store.getRecord(input.sessionId) + context: StructuredAgentSessionAttachContext + /** Who is asking; the attach keys the ledger row it settles by it. */ + callerKey: string + attachOptions?: StructuredAgentSessionAttachOptions +}): Promise { + const { sessionId, context, callerKey } = input + // The record is read only once this host has adjudicated it and exited any recovery stage a + // failed attempt latched — a lease left in `manual-recovery` by an unproven exit is one the + // resolver hands back, and the eligibility below must see it that way. + const unreconciled = await context.reconcileLeases(sessionId) + if (unreconciled) { + return { ok: false, refusal: unreconciled } + } + await context.runtimeState.resolveRecovery(sessionId) + const record = context.deps.store.getRecord(sessionId) if (!record) { - throw new Error('agent_session_identity_required') + return refuse('agent_session_identity_required', 'No structured session exists by that id.') } - if (!adapterSupportsRecord(input.deps.adapter, record)) { - throw new Error('structured_agent_session_unsupported') + if (!adapterSupportsRecord(context.deps.adapter, record)) { + return refuse( + 'structured_agent_session_unsupported', + 'This execution host cannot resume the requested structured agent session.' + ) } const params = structuredAgentSessionResumeParams( record, - structuredAgentSessionResumeOperationId(input.now()) + structuredAgentSessionResumeOperationId(context.now()) ) if (!params) { - throw new Error( - record.lease.unreconciled - ? 'execution_owner_reconciling' - : record.lease.claimStatus === 'conflicted' - ? 'agent_session_conflict' - : 'agent_session_ownership_unknown' + return record.lease.unreconciled + ? refuse( + 'execution_owner_reconciling', + 'This host has not yet adjudicated the session lease.' + ) + : record.lease.claimStatus === 'conflicted' + ? refuse('agent_session_conflict', 'Another process claims this session.') + : refuse( + 'agent_session_ownership_unknown', + 'The session lease is not one this host may resume.' + ) + } + let attached: AgentSessionMutationResult + try { + attached = await attachStructuredAgentSessionUnderSerialize( + context, + callerKey, + params, + input.attachOptions + ) + } catch (error) { + // The attach settles an acquisition that failed — the ledger row, the released lease — before + // it rethrows the cause. That row is the answer: a failure it recorded is this resume's + // refusal, verdict and all. Only an error it did not record is a fault for the caller. + const settled = settledResumeRefusal( + context, + callerKey, + params.envelope.clientOperationId, + sessionId, + error ) + if (settled) { + return settled + } + throw error } - const attached = await input.attach(params) - if (!attached.ok) { - throw new Error(attached.refusal.code) + return attached.ok ? { ok: true } : { ok: false, refusal: attached.refusal } +} + +function settledResumeRefusal( + context: Pick, + callerKey: string, + operationId: string, + sessionId: string, + error: unknown +): StructuredAgentSessionResumeOutcome | null { + const outcome = context.deps.store.getOperationRow(callerKey, operationId)?.outcome + if (outcome?.status !== 'failed' || !isAgentSessionWireRefusalCode(outcome.code)) { + return null } + return failedCreateRefusal( + { + code: outcome.code, + message: outcome.message ?? (error instanceof Error ? error.message : String(error)) + }, + outcome.status, + context.deps.store.getRecord(sessionId) + ) +} + +function refuse( + code: AgentSessionWireRefusal['code'], + message: string +): StructuredAgentSessionResumeOutcome { + return { ok: false, refusal: { code, message } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts index 268f28ed54e0..68027f81887c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts @@ -2,6 +2,7 @@ // deadline that keeps teardown from hanging. import { afterEach, describe, expect, it, vi } from 'vitest' +import { runKeyedSerializedOperation } from '../../cli/keyed-promise-queue' import { StructuredAgentSessionHolders } from './structured-agent-session-holders' import { StructuredAgentSessionReleaseClock } from './structured-agent-session-release-clock' import { StructuredAgentSessionHolds } from './structured-agent-session-holds' @@ -16,14 +17,21 @@ import { const clocks: StructuredAgentSessionReleaseClock[] = [] +/** The host's per-session queue, so a hold and a writer really take turns. */ +function keyedSerialize() { + const chains = new Map>() + return (sessionId: string, task: () => Promise) => + runKeyedSerializedOperation(chains, sessionId, task) +} + function clock(deps: { - isTurnActive?: () => boolean + hasOwedWork?: () => boolean isHeld?: () => boolean evict: (sessionId: string) => Promise onError?: (input: { sessionId: string; error: unknown }) => void }): StructuredAgentSessionReleaseClock { const created = new StructuredAgentSessionReleaseClock({ - isTurnActive: deps.isTurnActive ?? (() => false), + hasOwedWork: deps.hasOwedWork ?? (() => false), isHeld: deps.isHeld ?? (() => false), evict: deps.evict, ...(deps.onError ? { onError: deps.onError } : {}), @@ -89,7 +97,7 @@ describe('the release clock', () => { it('waits out a running turn instead of evicting into it', async () => { const evict = vi.fn(async () => {}) let turnRunning = true - const releasing = clock({ isTurnActive: () => turnRunning, evict }) + const releasing = clock({ hasOwedWork: () => turnRunning, evict }) releasing.arm('session-1') await new Promise((resolve) => setTimeout(resolve, 30)) @@ -109,6 +117,42 @@ describe('the release clock', () => { expect(evict).not.toHaveBeenCalled() }) + it('keeps an idle unheld child for thirty minutes, and activity starts the window over', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + try { + const evict = vi.fn(async () => {}) + const idle = new StructuredAgentSessionReleaseClock({ + hasOwedWork: () => false, + isHeld: () => false, + evict + }) + clocks.push(idle) + const minutes = (count: number) => vi.advanceTimersByTimeAsync(count * 60_000) + + idle.arm('session-1') + await minutes(20) + idle.renew('session-1') + await minutes(29) + expect(evict).not.toHaveBeenCalled() + + await minutes(1) + expect(evict).toHaveBeenCalledWith('session-1') + } finally { + vi.useRealTimers() + } + }) + + it('does not start a window for a session nothing released', async () => { + const evict = vi.fn(async () => {}) + const releasing = clock({ evict }) + + releasing.renew('session-1') + await new Promise((resolve) => setTimeout(resolve, 30)) + + expect(releasing.isArmed('session-1')).toBe(false) + expect(evict).not.toHaveBeenCalled() + }) + it('reports a failed eviction rather than swallowing it', async () => { const onError = vi.fn() const releasing = clock({ @@ -134,11 +178,13 @@ describe('holds', () => { let child = false const resume = vi.fn(async () => { child = true + return { ok: true as const } }) const holds = new StructuredAgentSessionHolds({ resume, + serialize: keyedSerialize(), hasProviderChild: () => child, - isTurnActive: () => false, + hasOwedWork: () => false, evict: async () => {}, graceMs: 1 }) @@ -155,12 +201,68 @@ describe('holds', () => { holds.dispose() }) + it('runs one resume for a writer and a hold that ask in the same gap', async () => { + const gate = Promise.withResolvers() + let child = false + const resume = vi.fn(async () => { + await gate.promise + child = true + return { ok: true as const } + }) + const serialize = keyedSerialize() + const holds = new StructuredAgentSessionHolds({ + resume, + serialize, + hasProviderChild: () => child, + hasOwedWork: () => false, + evict: async () => {}, + graceMs: 1 + }) + + // A send's ensure-owner step: already inside the session's serialize when it asks. + const writer = serialize('session-1', () => holds.ensureProviderChild('session-1')) + const hold = holds.hold('session-1', 'chat-1') + gate.resolve() + + await expect(writer).resolves.toEqual({ ok: true }) + await hold + // The hold ran after the writer's step and found the child: nothing to resume. + expect(resume).toHaveBeenCalledOnce() + // The surface arrived while the writer's resume ran, so the child it got is held, not idle. + expect(holds.isHeld('session-1')).toBe(true) + expect(holds.isReleasePending('session-1')).toBe(false) + holds.dispose() + }) + + it('puts a child a writer resumed with no surface on the idle clock', async () => { + let child = false + const serialize = keyedSerialize() + const holds = new StructuredAgentSessionHolds({ + resume: async () => { + child = true + return { ok: true as const } + }, + serialize, + hasProviderChild: () => child, + hasOwedWork: () => false, + evict: async () => {}, + graceMs: 60_000 + }) + + await serialize('session-1', () => holds.ensureProviderChild('session-1')) + + expect(holds.isHeld('session-1')).toBe(false) + expect(holds.isReleasePending('session-1')).toBe(true) + holds.dispose() + }) + it('never arms the clock for a session with nothing to stop', async () => { const evict = vi.fn(async () => {}) const holds = new StructuredAgentSessionHolds({ - resume: async () => {}, + resume: async () => ({ ok: true as const }), + serialize: keyedSerialize(), hasProviderChild: () => false, - isTurnActive: () => false, + hasOwedWork: () => false, evict, graceMs: 1 }) @@ -176,9 +278,10 @@ describe('holds', () => { it('fails a write-capable hold when resume proves no provider child', async () => { const holds = new StructuredAgentSessionHolds({ - resume: async () => {}, + resume: async () => ({ ok: true as const }), + serialize: keyedSerialize(), hasProviderChild: () => false, - isTurnActive: () => false, + hasOwedWork: () => false, evict: async () => {}, graceMs: 1 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts index 73880022646b..0acbccb6b55f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts @@ -6,22 +6,34 @@ // // A surface takes a hold when it binds and drops it when it goes away. The first hold on a session // with no child resumes it — that, and not the shape of a lease on disk, is what makes a provider -// process exist. The last hold leaving starts the release clock. Transport close is the BACKSTOP, +// process exist. The last hold leaving starts the idle release clock. Transport close is the BACKSTOP, // not the mechanism: a client that vanishes mid-flight never sends its release, so the caller // registers one against the connection and the holder set absorbs the duplicate. +// +// A send to a childless session resumes it too, and so does provider-exit recovery under an open +// surface. All three go through `ensureProviderChild` inside the session's serialize, so they take +// turns: the first to run attaches, and the next finds the child and attaches nothing. Two attaches +// for one session would race against the same released fence, and the loser's stale fence refused +// it — a hold that lost dropped its holder, a send that lost was refused. import { StructuredAgentSessionReleaseClock, type StructuredAgentSessionReleaseClockDeps } from './structured-agent-session-release-clock' import { StructuredAgentSessionHolders } from './structured-agent-session-holders' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' +import type { StructuredAgentSessionAttachOptions } from './structured-agent-session-attach-orchestration' export type StructuredAgentSessionHoldsDeps = { - /** Acquires a provider child for a session that has none. A no-op when one is already live. */ - resume: (sessionId: string) => Promise + /** Attaches a provider child, for a caller already inside `serialize`. */ + resume: ( + sessionId: string, + attachOptions?: StructuredAgentSessionAttachOptions + ) => Promise + serialize: (sessionId: string, task: () => Promise) => Promise /** Whether evicting this session would actually free anything. */ hasProviderChild: (sessionId: string) => boolean - isTurnActive: (sessionId: string) => boolean + hasOwedWork: (sessionId: string) => boolean evict: (sessionId: string) => Promise onError?: (input: { sessionId: string; error: unknown }) => void graceMs?: number @@ -40,7 +52,7 @@ export class StructuredAgentSessionHolds { constructor(private readonly deps: StructuredAgentSessionHoldsDeps) { const clockDeps: StructuredAgentSessionReleaseClockDeps = { - isTurnActive: deps.isTurnActive, + hasOwedWork: deps.hasOwedWork, isHeld: (sessionId) => this.holders.isHeld(sessionId), evict: (sessionId) => this.deps.evict(sessionId), ...(deps.onError ? { onError: deps.onError } : {}), @@ -63,23 +75,74 @@ export class StructuredAgentSessionHolds { if (options.resume === false) { return } + let resumed: StructuredAgentSessionResumeOutcome + try { + resumed = await this.deps.serialize(sessionId, () => this.ensureProviderChild(sessionId)) + } catch (error) { + this.releaseFailedHold(sessionId, holderId, alreadyHeld, incarnation) + throw error + } + if (!resumed.ok) { + this.releaseFailedHold(sessionId, holderId, alreadyHeld, incarnation) + // The RPC surface raises a refusal as its code. + throw new Error(resumed.refusal.code) + } + } + + /** Only the holder this call added, at the incarnation it added: a same-ID hold that left and + * came back while this one waited owns the holder now, and its own attempt decides it. */ + private releaseFailedHold( + sessionId: string, + holderId: string, + alreadyHeld: boolean, + incarnation: symbol | undefined + ): void { + if (!alreadyHeld && incarnation !== undefined) { + this.release(sessionId, holderId, incarnation) + } + } + + /** + * Gives the session a provider child if it has none. + * + * For a caller already inside the session's serialize, which is what makes "if it has none" + * exact: a hold and a send that both find the owner gone run this in turn, and the second sees + * the first one's child. Each caller makes at most one attach, and a failed one leaves the + * next caller to make its own. With no surface holding the session afterwards, the child goes + * on the same clock a departed surface would start — including the surface that held it when + * provider-exit recovery began and left while the attach ran. + */ + async ensureProviderChild( + sessionId: string, + attachOptions?: StructuredAgentSessionAttachOptions + ): Promise { + if (this.deps.hasProviderChild(sessionId)) { + return { ok: true } + } + const resumed = await this.deps.resume(sessionId, attachOptions) + if (!resumed.ok) { + return resumed + } if (!this.deps.hasProviderChild(sessionId)) { - try { - await this.deps.resume(sessionId) - if (!this.deps.hasProviderChild(sessionId)) { - throw new Error('agent_session_ownership_unknown') - } - // The last surface can disconnect before acquisition makes a child available to release. - if (!this.disposed && !this.holders.isHeld(sessionId)) { - this.clock.arm(sessionId) + return { + ok: false, + refusal: { + code: 'agent_session_ownership_unknown', + message: 'The session attached without a provider child to write to.' } - } catch (error) { - if (!alreadyHeld && incarnation !== undefined) { - this.release(sessionId, holderId, incarnation) - } - throw error } } + // The last surface can disconnect before acquisition makes a child available to release. + if (!this.disposed && !this.holders.isHeld(sessionId)) { + this.clock.arm(sessionId) + } + return { ok: true } + } + + /** Activity — a journal write, or a start reaching the work it held; only a pending release + * notices, and it restarts its full window. */ + renew(sessionId: string): void { + this.clock.renew(sessionId) } release(sessionId: string, holderId: string, expectedIncarnation?: symbol): void { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.test.ts index bf2a1381b3b6..55e186ba5232 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.test.ts @@ -10,13 +10,17 @@ import type { AgentSessionHandoffStatus } from '../../../shared/agent-session-wi import { LOCAL_EXECUTION_HOST_ID } from '../../../shared/execution-host' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' -import { createDeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import { + createDeferredStructuredAgentSessionEventSink, + type DeferredStructuredAgentSessionEventSink +} from './structured-agent-session-event-sink' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { - acquireNativeHandoffOwner, createStructuredAgentSessionHostHandoff, structuredTuiTranscriptImportOptions } from './structured-agent-session-host-handoff' +import { acquireNativeHandoffOwner } from './structured-agent-session-native-handoff-acquisition' +import { AgentSessionSubscribers } from './structured-agent-session-subscribers' const journals = createTrackedJournalOpener() @@ -30,6 +34,19 @@ function importRecord(provider: 'claude' | 'codex', accountHome: string): AgentS } as AgentSessionRecord } +/** The session's sink is `current`; a native acquire's attempt gets its own. */ +function sinksOver(current: DeferredStructuredAgentSessionEventSink) { + return { + eventSinkFor: () => current, + mintEventSink: () => createDeferredStructuredAgentSessionEventSink(), + adoptEventSink: () => undefined + } +} + +function unreachableSink(): never { + throw new Error('unreachable: publishing reads no sink') +} + describe('structured TUI transcript import roots', () => { it('uses the managed Claude account home when no live transcript path remains', () => { expect(structuredTuiTranscriptImportOptions(importRecord('claude', '/managed/claude'))).toEqual( @@ -165,7 +182,10 @@ describe('native handoff acquisition', () => { }, fence: reserved.record.lease.runtimeFence, hasProviderChild: false, - acquisitionGeneration: null + providerChildPhase: 'ready' as const, + acquisitionGeneration: null, + // Left by a restart before the handoff; a writer current as of it is not current now. + resumedFromFence: 1 } const acquiring = acquireNativeHandoffOwner( { @@ -177,7 +197,7 @@ describe('native handoff acquisition', () => { { session: () => session, findSession: () => session, - eventSink: () => eventSink, + eventSinks: sinksOver(eventSink), flush: async () => undefined, serialize: async (_session, task) => task(), subscribers: { @@ -202,6 +222,8 @@ describe('native handoff acquisition', () => { await acquiring expect(order).toEqual(['append-entered', 'append-complete', 'unbind', 'acquire']) + // The handoff moved the fence, not a restart: nothing is rebased across it. + expect(session.resumedFromFence).toBeUndefined() }) it('refuses an unsupported adapter before unbinding the TUI owner', async () => { @@ -262,6 +284,7 @@ describe('native handoff acquisition', () => { }, fence: reserved.record.lease.runtimeFence, hasProviderChild: false, + providerChildPhase: 'ready' as const, acquisitionGeneration: null } @@ -276,7 +299,7 @@ describe('native handoff acquisition', () => { { session: () => session, findSession: () => session, - eventSink: () => eventSink, + eventSinks: sinksOver(eventSink), flush: async () => undefined, serialize: async (_sessionId, task) => task(), subscribers: { @@ -356,6 +379,7 @@ describe('native handoff acquisition', () => { }, fence: reserved.record.lease.runtimeFence, hasProviderChild: false, + providerChildPhase: 'ready' as const, acquisitionGeneration: null } @@ -370,7 +394,7 @@ describe('native handoff acquisition', () => { { session: () => session, findSession: () => session, - eventSink: () => eventSink, + eventSinks: sinksOver(eventSink), flush: async () => undefined, serialize: async (_sessionId, task) => task(), subscribers: { @@ -388,6 +412,103 @@ describe('native handoff acquisition', () => { expect(unbind).toHaveBeenCalledOnce() expect(acquire).not.toHaveBeenCalled() }) + + it("takes a failed native child's queued rows with it, leaving the session sink drainable", async () => { + const sessionId = 'session-handoff-failed' + const location: AgentSessionExecutionLocation = { + executionHostId: LOCAL_EXECUTION_HOST_ID, + wslDistro: null, + workspaceId: 'workspace-failed', + workspaceKind: 'folder' + } + const operationId = `${now}-00000000000000000000000000000031` + const reserved = await store.reserveOwner({ + sessionId, + location, + provider: 'codex', + accountHome: { variable: 'CODEX_HOME', path: join(root, 'codex-home') }, + runtimeKind: 'native', + expectedFence: null, + spawnToken: 'failed-spawn', + claimKeyId: 'key-1', + handoffOperationId: operationId, + probe: { outcome: 'reservation-unused' }, + operation: { callerKey: 'test', operationId, fingerprint: 'failed' }, + now + }) + const journal = await journals.open({ + identity: { + sessionId, + workspaceId: location.workspaceId, + hostId: location.executionHostId, + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'failed-thread' } + }, + journalDir: join(root, 'failed-journal') + }) + const current = createDeferredStructuredAgentSessionEventSink() + current.bind({ journal, fence: reserved.record.lease.runtimeFence, publish: () => undefined }) + const attempt = createDeferredStructuredAgentSessionEventSink() + const acquire = vi.fn>( + async ({ events }) => { + // The child wrote before it died, into whatever sink it was handed. + events?.setActivity?.(null) + throw new Error('codex app-server exited (code 1)') + } + ) + const session = { + journal, + params: { + envelope: { + sessionId, + clientOperationId: `${now}-00000000000000000000000000000032`, + expectedRuntimeFence: reserved.record.lease.runtimeFence, + payloadFingerprint: 'failed' + }, + location, + provider: 'codex' as const, + agent: 'codex' as const, + accountHome: { variable: 'CODEX_HOME' as const, path: join(root, 'codex-home') }, + runtimeKind: 'native' as const, + providerHandle: { kind: 'codex' as const, threadId: 'failed-thread' } + }, + fence: reserved.record.lease.runtimeFence, + hasProviderChild: false, + providerChildPhase: 'ready' as const, + acquisitionGeneration: null + } + + await expect( + acquireNativeHandoffOwner( + { + store, + adapter: { + acquire, + dispatch: vi.fn(async () => ({ state: 'admitted' as const })), + cancelTurn: vi.fn(async () => ({ cancelled: true })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined) + }, + journalRoot: root, + claimKeyId: 'key-1' + }, + { + session: () => session, + findSession: () => session, + eventSinks: { ...sinksOver(current), mintEventSink: () => attempt }, + flush: async () => undefined, + serialize: async (_sessionId, task) => task(), + subscribers: new AgentSessionSubscribers(), + now: () => now + }, + { sessionId, fence: reserved.record.lease.runtimeFence, spawnToken: 'failed-spawn' } + ) + ).rejects.toThrow('codex app-server exited') + // The next attach drains the session's sink before acquiring; nothing may be stranded there. + expect(current.state().queuedOperations).toBe(0) + await expect(current.drained()).resolves.toEqual({ ok: true }) + expect(attempt.sink.tryPublish?.()).toEqual({ accepted: false, reason: 'closed' }) + }) }) describe('handoff status published for a session the host no longer holds', () => { @@ -421,8 +542,10 @@ describe('handoff status published for a session the host no longer holds', () = throw new Error('agent_session_ownership_unknown') }, findSession: () => undefined, - eventSink: () => { - throw new Error('unreachable: publishing reads no sink') + eventSinks: { + eventSinkFor: unreachableSink, + mintEventSink: unreachableSink, + adoptEventSink: unreachableSink }, flush: async () => undefined, serialize: async (_sessionId, task) => task(), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.ts index 2720a97784c8..41e2d8c61e3f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-handoff.ts @@ -3,29 +3,27 @@ import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { LegacyImportOptions } from '../agent-session-journal/journal-legacy-import' import { importLegacyTranscriptIntoJournal } from '../agent-session-journal/journal-legacy-import' -import { journalIdentityFor } from './structured-agent-session-attach' -import { - rethrowAfterAgentSessionAcquisitionCleanup, - type StructuredAgentSessionAdapter -} from './structured-agent-session-adapter' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { canRestoreLiveTuiOwner } from './structured-agent-session-handoff-restart' -import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host' +import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { StructuredAgentSessionHandoffCoordinator } from './structured-agent-session-handoff' import { recoverDeadTuiHandoffStatus } from './structured-agent-session-dead-tui-recovery' -import { readNativeSessionOptions } from './structured-agent-session-option-restoration' +import { acquireNativeHandoffOwner } from './structured-agent-session-native-handoff-acquisition' import type { AgentSessionSubscribers } from './structured-agent-session-subscribers' import { StructuredTuiTranscriptCatchup } from './structured-tui-transcript-catchup' -import { adapterSupportsCreateIfDeclared } from './structured-agent-session-provider-support' import { retryLoadedStructuredAgentSessionSettlement } from './structured-agent-session-settlement-retry' import { latestJournalDispatchObservation } from '../agent-session-journal/journal-dispatch-observation' -type HostHandoffAccess = { +export type HostHandoffAccess = { session: (sessionId: string) => StructuredAgentSessionHostSession /** Non-throwing lookup, for the paths that only observe a detached session. */ findSession: (sessionId: string) => StructuredAgentSessionHostSession | undefined - eventSink: (sessionId: string) => DeferredStructuredAgentSessionEventSink + eventSinks: Pick< + StructuredAgentSessionHostRuntimeState, + 'eventSinkFor' | 'mintEventSink' | 'adoptEventSink' + > flush: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise subscribers: AgentSessionSubscribers @@ -97,7 +95,7 @@ export function createStructuredAgentSessionHostHandoff( ) host.subscribers.publish(sessionId, session.journal) host.publishStatus?.(sessionId) - host.eventSink(sessionId).unbind() + host.eventSinks.eventSinkFor(sessionId).unbind() return { state: 'stopped' } } catch (error) { return { state: 'stopped-cleanup-failed', error } @@ -217,76 +215,3 @@ export function structuredTuiTranscriptImportOptions( ? { claudeProjectsDir: join(record.accountHome.path, 'projects') } : { codexSessionsDirs: [join(record.accountHome.path, 'sessions')] } } - -export async function acquireNativeHandoffOwner( - deps: StructuredAgentSessionHostDeps, - host: HostHandoffAccess, - input: { sessionId: string; fence: number; spawnToken: string } -): Promise { - const session = host.session(input.sessionId) - const record = deps.store.getRecord(input.sessionId) - if (!record) { - throw new Error('agent_session_identity_required') - } - // Native handoff bypasses attach admission; reject before unbinding TUI ownership. - if (!adapterSupportsCreateIfDeclared(deps.adapter, record.location, record.provider)) { - throw new Error('structured_agent_session_unsupported') - } - const eventSink = host.eventSink(input.sessionId) - const priorBarrier = await eventSink.drained() - if (!priorBarrier.ok) { - throw priorBarrier.error - } - eventSink.unbind() - // Recheck immediately before acquisition; capability probes may drift while - // the old TUI event sink is draining. - if (!adapterSupportsCreateIfDeclared(deps.adapter, record.location, record.provider)) { - throw new Error('structured_agent_session_unsupported') - } - const acquired = await deps.adapter.acquire({ - identity: journalIdentityFor(record, session.params), - fence: input.fence, - spawnToken: input.spawnToken, - ...(record.options ? { options: record.options } : {}), - events: eventSink.sink - }) - let proved: AgentSessionRecord - try { - const options = await readNativeSessionOptions({ - adapter: deps.adapter, - sessionId: input.sessionId, - fence: input.fence, - ...(record.options ? { priorOptions: record.options } : {}) - }) - await deps.store.commitProcessIdentity({ - sessionId: input.sessionId, - fence: input.fence, - process: acquired.process, - now: host.now() - }) - proved = await deps.store.proveOwner({ - sessionId: input.sessionId, - fence: input.fence, - link: acquired.link, - now: host.now(), - ...(options ? { options } : {}) - }) - } catch (error) { - return rethrowAfterAgentSessionAcquisitionCleanup(deps.adapter, input.sessionId, error) - } - session.hasProviderChild = true - host.publishStatus?.(input.sessionId) - session.fence = proved.lease.runtimeFence - session.acquisitionGeneration = acquired.acquisitionGeneration ?? null - eventSink.bind({ - journal: session.journal, - fence: proved.lease.runtimeFence, - publish: (activity) => host.subscribers.publish(input.sessionId, session.journal, activity) - }) - const acquiredBarrier = await eventSink.drained() - if (!acquiredBarrier.ok) { - throw acquiredBarrier.error - } - host.subscribers.snapshot(input.sessionId, session.journal, proved.lease.runtimeFence) - return proved -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 4c6b355214a4..aee8a8825766 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -20,7 +20,7 @@ import type { } from './structured-agent-session-host-types' import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' import { resumeHeldStructuredAgentSession } from './structured-agent-session-hold-resume' -import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' export type StructuredAgentSessionLifetimeContext = { @@ -165,48 +165,39 @@ export async function evictOwnedStructuredAgentSessions( } } -/** The first hold on a childless session: reconcile the lease, settle recovery, then attach. */ -export async function resumeStructuredAgentSessionForHold( - context: StructuredAgentSessionLifetimeContext & { - reconcileLeases: (sessionId: string) => Promise - }, - sessionId: string, - attach: Parameters[0]['attach'] -): Promise { - const unreconciled = await context.reconcileLeases(sessionId) - if (unreconciled) { - throw new Error(unreconciled.code) - } - await context.runtimeState.resolveRecovery(sessionId) - await resumeHeldStructuredAgentSession({ - sessionId, - deps: context.deps, - now: context.now, - attach - }) -} - +/** The holds resume through the host's own attach, inside the session's serialize: a hold's + * resume and a send's ensure-owner step are the same serialized attach with a different asker. */ export function createStructuredAgentSessionHolds( - context: StructuredAgentSessionLifetimeContext, - input: { - reconcileLeases: (sessionId: string) => Promise - attach: Parameters[0]['attach'] - close: (sessionId: string) => Promise - } + attachContext: () => StructuredAgentSessionAttachContext, + close: (sessionId: string) => Promise ): StructuredAgentSessionHolds { + const context = attachContext() return new StructuredAgentSessionHolds({ - resume: (sessionId) => - resumeStructuredAgentSessionForHold( - { ...context, reconcileLeases: input.reconcileLeases }, + resume: (sessionId, attachOptions) => + resumeHeldStructuredAgentSession({ sessionId, - input.attach - ), - evict: input.close, + context: attachContext(), + callerKey: attachOptions?.admitRecoveryTicket + ? 'trusted-local:provider-exit-recovery' + : 'trusted-local:surface-hold', + ...(attachOptions ? { attachOptions } : {}) + }), + // Tracked from enqueue: a quit drains a queued resume before it evicts, so no child is + // spawned behind the eviction and orphaned. + serialize: (sessionId, task) => { + const current = attachContext() + return current.tasks.trackAttach(current.serialize(sessionId, task)) + }, + evict: close, hasProviderChild: (sessionId) => hasProviderChild(context, sessionId), - isTurnActive: (sessionId) => { + // A send pending while the child is still starting is held for that start; evicting would + // refuse it. Any other pending send may wait on an echo that never comes, so eviction retires it. + hasOwedWork: (sessionId) => { const session = context.sessions.get(sessionId) return session - ? activeStructuredAgentSessionTurnId(session.journal.snapshot().items) !== null + ? activeStructuredAgentSessionTurnId(session.journal.snapshot().items) !== null || + (session.providerChildPhase === 'starting' && + session.journal.pendingSubmissions().length > 0) : false }, onError: (error) => context.deps.onEventSinkError?.(error), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index 2374181554a1..4773ea5eab8d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -1,10 +1,10 @@ -import { rewindRefusal } from './structured-rewind-refusal' -// Everything a client can ask an ALREADY-ATTACHED session to do: send a turn, cancel one, answer a -// prompt, change an option, read the options back. +// Everything a client can ask an ATTACHED session to do: send a turn, cancel one, answer a prompt, +// change an option, read the options back. // // They share one shape — admit the envelope against the lease, run a plan, publish the journal — so // they share one path here rather than five copies in the host. The host keeps attach, holds and -// teardown; this is the surface that assumes those already happened. +// teardown. A send is the one mutation that may need those first: it makes sure the session has +// an owner as a step of its own serialized admission, see `structured-agent-session-send-preparation`. import type { AgentJournalItemIdentity, @@ -21,8 +21,16 @@ import type { AgentSessionThreadGoalChange, AgentSessionThreadGoalResult } from '../../../shared/agent-session-wire' +import type { StructuredAgentSessionHolds } from './structured-agent-session-holds' import { threadGoalPlan } from './structured-agent-session-thread-goal' -import { admitAndRunAgentSessionMutation } from './structured-agent-session-mutation-admission' +import { + admitAndRunAgentSessionMutation, + type AgentSessionMutationRequest +} from './structured-agent-session-mutation-admission' +import { + prepareStructuredAgentSessionSend, + structuredAgentSessionSendBlock +} from './structured-agent-session-send-preparation' import { cancelPlan, promptPlan, @@ -44,6 +52,11 @@ export type StructuredAgentSessionMutationContext = { hasPendingStreamedEvents?: (sessionId: string) => boolean requireSession: (sessionId: string) => StructuredAgentSessionHostSession serialize: (sessionId: string, task: () => Promise) => Promise + /** A send that finds the owner gone brings it back through here, inside its own serialize. */ + holds: Pick + /** Makes a closed session's journal readable again, inside the caller's serialize, for a send + * the ledger answers without an owner. */ + restoreReadable: (sessionId: string) => Promise now: () => number } @@ -51,7 +64,8 @@ function mutate( context: StructuredAgentSessionMutationContext, caller: StructuredAgentSessionCaller, envelope: AgentSessionMutationEnvelope, - plan: MutationPlan + plan: MutationPlan, + prepareSession?: AgentSessionMutationRequest['prepareSession'] ): Promise> { return context.serialize(envelope.sessionId, () => admitAndRunAgentSessionMutation({ @@ -60,10 +74,12 @@ function mutate( callerKey: caller.callerKey, envelope, plan, - journal: context.sessions.get(envelope.sessionId)?.journal, + journal: () => context.sessions.get(envelope.sessionId)?.journal, + prepareSession, publish: (journal) => context.publish(envelope.sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, hasPendingStreamedEvents: context.hasPendingStreamedEvents, + providerChildPhase: () => context.sessions.get(envelope.sessionId)?.providerChildPhase, now: () => context.now() }) ) @@ -80,32 +96,19 @@ export function sendStructuredAgentSessionTurn( } ): Promise> { const plan = sendPlan(params) - return mutate(context, caller, params.envelope, { - ...plan, - run: (ctx) => { - const rewind = context.deps.store.getRecord(ctx.sessionId)?.rewind - if (rewind?.phase === 'prepared' || rewind?.phase === 'provider-succeeded') { - return Promise.resolve(rewindRefusal('outcome-unknown')) - } - const command = context.deps.store.getRecord(ctx.sessionId)?.conversationCommand - if ( - command && - ((command.state === 'unknown' && command.phase === 'prepared') || - (command.command === 'clear' && command.replacementSessionId)) - ) { - return Promise.resolve({ - ok: false, - refusal: { - code: 'agent_session_operation_invalid', - message: command.replacementSessionId - ? 'This conversation has been cleared. Use the current conversation.' - : 'The conversation operation is unconfirmed.' - } - }) + return mutate( + context, + caller, + params.envelope, + { + ...plan, + run: (ctx) => { + const blocked = structuredAgentSessionSendBlock(context.deps.store.getRecord(ctx.sessionId)) + return blocked ? Promise.resolve(blocked) : plan.run(ctx) } - return plan.run(ctx) - } - }) + }, + (ledger, record) => prepareStructuredAgentSessionSend(context, params.envelope, ledger, record) + ) } export function cancelStructuredAgentSessionTurn( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts index 85eaf1e29410..e0ea81c6b056 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts @@ -42,30 +42,54 @@ export class StructuredAgentSessionHostRuntimeState { this.leaseRenewer.stop() } + /** The sink the session's current child writes through, created on first use. */ eventSinkFor(sessionId: string): DeferredStructuredAgentSessionEventSink { - const existing = this.eventSinks.get(sessionId) + const existing = this.currentEventSink(sessionId) if (existing) { - // A sink failure is terminal for that sink instance. Reusing it on a - // recovery attach makes `drained()` return the old error forever and - // prevents the newly acquired journal from accepting provider events. - // Replace the cache entry before attach calls its drain barrier. - if (existing.state().failed) { - existing.close() - this.eventSinks.delete(sessionId) - } else { - return existing - } + return existing } - const created = createDeferredStructuredAgentSessionEventSink({ - onError: (error) => { - this.deps.onEventSinkError?.({ sessionId, error }) - this.onEventSinkFailure?.(sessionId, error) - } - }) + const created = this.mintEventSink(sessionId) this.eventSinks.set(sessionId, created) return created } + /** The session's sink, if it has a usable one. A failed sink is terminal: reusing it would make + * `drained()` return the old error forever, so it is dropped here instead. */ + currentEventSink(sessionId: string): DeferredStructuredAgentSessionEventSink | undefined { + const existing = this.eventSinks.get(sessionId) + if (existing?.state().failed) { + existing.close() + this.eventSinks.delete(sessionId) + return undefined + } + return existing + } + + /** A sink owned by one attach attempt. Uncached until `adoptEventSink`, so an attempt that fails + * takes its queue with it rather than leaving it for the next attach to drain. */ + mintEventSink(sessionId: string): DeferredStructuredAgentSessionEventSink { + const minted: DeferredStructuredAgentSessionEventSink = + createDeferredStructuredAgentSessionEventSink({ + onError: (error) => { + this.deps.onEventSinkError?.({ sessionId, error }) + // Only the session's own sink may force its provider down; an attempt's never is. + if (this.eventSinks.get(sessionId) === minted) { + this.onEventSinkFailure?.(sessionId, error) + } + } + }) + return minted + } + + /** The attempt's sink now serves the session; the one it replaces is closed. */ + adoptEventSink(sessionId: string, sink: DeferredStructuredAgentSessionEventSink): void { + const replaced = this.eventSinks.get(sessionId) + if (replaced && replaced !== sink) { + replaced.close() + } + this.eventSinks.set(sessionId, sink) + } + discardEventSink(sessionId: string): void { this.eventSinks.delete(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index 8d16a0a33467..a3d207e72b9a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -6,7 +6,10 @@ import type { AgentSessionRecordStore } from '../../runtime/agent-session-record import type { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' import type { AgentSessionSpawnTokenScan } from '../../runtime/agent-session-spawn-token-process-scan' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import type { + StructuredAgentSessionAdapter, + StructuredAgentSessionProviderChildPhase +} from './structured-agent-session-adapter' import type { AgentSessionAttachParams } from './structured-agent-session-attach' import type { StructuredAgentSessionHandoffTransport } from './structured-agent-session-handoff-types' import type { StructuredAgentSessionStatusSink } from './structured-agent-session-status-feed' @@ -32,6 +35,9 @@ export type StructuredAgentSessionHostSession = { * restored for reading has none, and neither has a session a TUI owns — so neither may be * evicted to free a child, and neither may have its lease released as an observed exit. */ hasProviderChild: boolean + /** Whether the child behind `hasProviderChild` has proven its start. A publish-first acquire + * is `starting` until the adapter's `started` event; only then are its reported options fact. */ + providerChildPhase: StructuredAgentSessionProviderChildPhase /** The wind-down this host still owes for a child it started: settling that generation's work * and handing the lease back. A separate fact from `hasProviderChild`, which goes false the * moment the adapter proves the exit — an eviction that aborts after that point must still be @@ -39,6 +45,10 @@ export type StructuredAgentSessionHostSession = { owesProviderChildWindDown?: boolean /** Exact adapter acquisition behind `hasProviderChild`; retained after exit to fence recovery. */ acquisitionGeneration: string | null + /** The fence of the released owner this child replaced, when it was resumed into a lease handed + * back cleanly. A writer current as of that owner is admitted at `fence`: the restart is the + * only thing that moved it. Absent for a create, a handoff, or a journal restored for reading. */ + resumedFromFence?: number } export type StructuredAgentSessionHostDeps = { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts index cbf7b95590f8..7426bcfba2ef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts @@ -138,13 +138,16 @@ describe('attach', () => { }) const params = attachParams() - await expect(host.attach(CALLER, params)).rejects.toThrow( - 'agent_session_provider_handle_stale_fence' - ) - expect(await host.attach(CALLER, params)).toMatchObject({ + const refused = { ok: false, - refusal: { code: 'agent_session_operation_invalid' } - }) + refusal: { + code: 'agent_session_operation_invalid', + message: 'agent_session_provider_handle_stale_fence', + ownerVerdict: 'exited' + } + } + expect(await host.attach(CALLER, params)).toEqual(refused) + expect(await host.attach(CALLER, params)).toEqual(refused) const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 expect(await host.attach(CALLER, ensureParams(releasedFence))).toMatchObject({ ok: true }) expect(acquire).toHaveBeenCalledTimes(2) @@ -155,7 +158,10 @@ describe('attach', () => { it('reaps an acquisition when process identity commit fails', async () => { vi.spyOn(store, 'commitProcessIdentity').mockRejectedValueOnce(new Error('commit failed')) - await expect(host.attach(CALLER, attachParams())).rejects.toThrow('commit failed') + await expect(host.attach(CALLER, attachParams())).resolves.toMatchObject({ + ok: false, + refusal: { message: 'commit failed', ownerVerdict: 'exited' } + }) expect(releaseAcquisition).toHaveBeenCalledWith({ sessionId: SESSION }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index b439f880c900..a2b03e120cac 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -66,7 +66,8 @@ export class StructuredAgentSessionHost { private readonly clientDelivery = new StructuredAgentSessionClientDelivery( this.sessions, () => this.now(), - () => this.deps + () => this.deps, + (sessionId) => this.holds.renew(sessionId) ) private readonly subscribers = this.clientDelivery.subscribers private readonly tasks = new StructuredAgentSessionTaskQueue() @@ -111,18 +112,17 @@ export class StructuredAgentSessionHost { this.handoffs = createStructuredAgentSessionHostHandoff(deps, { session: (sessionId) => this.requireSession(sessionId), findSession: (sessionId) => this.sessions.get(sessionId), - eventSink: (sessionId) => this.runtimeState.eventSinkFor(sessionId), + eventSinks: this.runtimeState, flush: (sessionId) => this.flushStreamedEvents(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), subscribers: this.subscribers, publishStatus: this.clientDelivery.publishStatus, now: this.now }) - this.holds = createStructuredAgentSessionHolds(this.lifetimeContext(), { - reconcileLeases: this.reconcileLeases, - attach: (params) => this.attach({ callerKey: 'trusted-local:surface-hold' }, params), - close: (sessionId) => this.close(sessionId) - }) + this.holds = createStructuredAgentSessionHolds( + () => this.attachContext(), + (sessionId) => this.close(sessionId) + ) this.restore = createStructuredAgentSessionHostRestore(deps, this.sessions, () => this.now(), { reconcile: this.reconcileLeases, resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), @@ -145,9 +145,11 @@ export class StructuredAgentSessionHost { this.subscribers.snapshot(sessionId, session.journal, session.fence), publishStatus: this.clientDelivery.publishStatusAndSettlement, hasResumeCapableHolder: (sessionId) => this.holds.hasResumeCapableHolder(sessionId), - serialize: (sessionId, task) => this.serialize(sessionId, task), + restartReleaseGrace: (sessionId) => this.holds.renew(sessionId), + // Tracked: a quit drains a queued restart before it evicts, so no child outlives it. + serialize: (sessionId, task) => this.tasks.trackAttach(this.serialize(sessionId, task)), now: () => this.now(), - attachContext: () => this.attachContext(), + ensureProviderChild: (id, options) => this.holds.ensureProviderChild(id, options), onBarrierError: (sessionId, error) => deps.onEventSinkError?.({ sessionId, error }) }) this.restartResume = createStructuredAgentSessionRestartResume(deps, this.sessions, { @@ -170,7 +172,7 @@ export class StructuredAgentSessionHost { options?: StructuredAgentSessionHoldOptions ): Promise => this.holds.hold(sessionId, holderId, options) - /** That surface is gone. The child outlives it by the release grace, and by any running turn. */ + /** That surface is gone. The child outlives it by the idle window, and by any running turn. */ release = (sessionId: string, holderId: string): void => this.holds.release(sessionId, holderId) handleAdapterEvent = (event: Parameters[0]) => @@ -204,8 +206,7 @@ export class StructuredAgentSessionHost { await this.handoffs.closeRetainedTuiOwner(sessionId) await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId) this.clientDelivery.closeSession(sessionId) - // Whoever asked for the close, the surfaces that were holding this session are looking at a - // session that no longer exists. A failed eviction throws above and keeps them. + // The holders now look at a session that is gone; a failed eviction throws above, keeping them. this.holds.forget(sessionId) }) } @@ -273,16 +274,16 @@ export class StructuredAgentSessionHost { sessions: this.sessions, publish: (sessionId, journal) => this.subscribers.publish(sessionId, journal), flushStreamedEvents: this.flushStreamedEvents, - hasPendingStreamedEvents: (sessionId) => - this.runtimeState.hasPendingStreamedEvents(sessionId), + hasPendingStreamedEvents: (id) => this.runtimeState.hasPendingStreamedEvents(id), requireSession: (sessionId) => this.requireSession(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), + holds: this.holds, + restoreReadable: (sessionId) => this.restore.restoreReadableUnderSerialize(sessionId), now: () => this.now() } } - send = (...args: Parameters) => - this.conversationCommands.send(...args) + send = this.conversationCommands.send waitForSendSettlement = this.clientDelivery.waitForSendSettlement diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts index ea32f1715702..1c9f271004c3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts @@ -75,6 +75,7 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe params: {} as StructuredAgentSessionHostSession['params'], fence: 1, hasProviderChild: false, + providerChildPhase: 'ready', acquisitionGeneration: null } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index ca1ddb7b8da0..b735427be0e8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -2,12 +2,19 @@ // fingerprint, admit through the durable operation ledger, check the lease, then // run the plan. It lives outside the host so that no method can quietly grow its // own admission rules by sitting next to the call site. +// +// Admission is two-phase for a call that brings a `prepareSession`. The ledger's +// answer comes first and places nothing; a call it will admit may then give the +// session an owner, and only after that are the row placed and the lease and +// fence checked — against the lease as it stands once the owner is there. import { admitAgentSessionMutation, agentSessionFingerprintConflict, computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionOperationDecision } from '../../../shared/agent-session-operation-ledger' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionMutationEnvelope, AgentSessionMutationResult, @@ -36,27 +43,37 @@ export function refuseAgentSessionMutation(refusal: AgentSessionWireRefusal): { return { ok: false, refusal } } +export type AgentSessionMutationSessionPreparation = + | { ok: true; envelope: AgentSessionMutationEnvelope } + | { ok: false; refusal: AgentSessionWireRefusal } + export type AgentSessionMutationRequest = { store: AgentSessionRecordStore adapter: StructuredAgentSessionAdapter callerKey: string envelope: AgentSessionMutationEnvelope plan: MutationPlan - /** Journal of the attached session; absent when this host holds none. */ - journal: AgentSessionJournal | undefined + /** Journal of the attached session, read after `prepareSession`; absent when this host holds none. */ + journal: () => AgentSessionJournal | undefined + /** Between the ledger's answer and the lease check, for a call that may first have to make the + * session ready for itself. Answers with the envelope to admit — the caller's, or one moved + * onto a fence the preparation itself published — or with the refusal that ends the call. */ + prepareSession?: ( + ledger: Exclude, + record: AgentSessionRecord + ) => Promise publish: (journal: AgentSessionJournal) => void flushStreamedEvents: (sessionId: string) => Promise hasPendingStreamedEvents?: (sessionId: string) => boolean + providerChildPhase?: AgentSessionTurnContext['providerChildPhase'] now: () => number } export async function admitAndRunAgentSessionMutation( request: AgentSessionMutationRequest ): Promise> { - const { envelope, plan, journal } = request - if (!journal) { - return refuseAgentSessionMutation(AGENT_SESSION_NOT_ATTACHED) - } + const { plan } = request + let { envelope } = request const hostFingerprint = computeAgentSessionPayloadFingerprint({ method: plan.method, sessionId: envelope.sessionId, @@ -66,6 +83,29 @@ export async function admitAndRunAgentSessionMutation( if (conflict) { return refuseAgentSessionMutation(conflict) } + if (request.prepareSession) { + const ledger = request.store.evaluateMutationOperation({ + callerKey: request.callerKey, + envelope, + hostFingerprint, + now: request.now(), + ...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}) + }) + if (!ledger) { + return refuseAgentSessionMutation(AGENT_SESSION_NOT_ATTACHED) + } + if (ledger.decision.decision !== 'refused') { + const prepared = await request.prepareSession(ledger.decision.decision, ledger.record) + if (!prepared.ok) { + return prepared + } + envelope = prepared.envelope + } + } + const journal = request.journal() + if (!journal) { + return refuseAgentSessionMutation(AGENT_SESSION_NOT_ATTACHED) + } const admitted = await request.store.admitMutationOperation({ callerKey: request.callerKey, envelope, @@ -152,6 +192,7 @@ function turnContext( flushStreamedEvents: () => request.flushStreamedEvents(request.envelope.sessionId), hasPendingStreamedEvents: () => request.hasPendingStreamedEvents?.(request.envelope.sessionId) ?? false, + ...(request.providerChildPhase ? { providerChildPhase: request.providerChildPhase } : {}), now: () => request.now() } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-native-handoff-acquisition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-native-handoff-acquisition.ts new file mode 100644 index 000000000000..c88f0918d25e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-native-handoff-acquisition.ts @@ -0,0 +1,133 @@ +// The host's half of a TUI-to-native handoff acquiring its native child. +// +// Like attach, the new child writes through a sink this attempt owns: only a proven owner makes it +// the session's, so a failed acquire takes whatever its child queued with it instead of leaving it +// in the session's sink for the next drain to wait on. + +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { journalIdentityFor } from './structured-agent-session-attach' +import { rethrowAfterAgentSessionAcquisitionCleanup } from './structured-agent-session-adapter' +import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host' +import type { HostHandoffAccess } from './structured-agent-session-host-handoff' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { readNativeSessionOptions } from './structured-agent-session-option-restoration' +import { adapterSupportsCreateIfDeclared } from './structured-agent-session-provider-support' + +export async function acquireNativeHandoffOwner( + deps: StructuredAgentSessionHostDeps, + host: HostHandoffAccess, + input: { sessionId: string; fence: number; spawnToken: string } +): Promise { + const session = host.session(input.sessionId) + const record = deps.store.getRecord(input.sessionId) + if (!record) { + throw new Error('agent_session_identity_required') + } + // Native handoff bypasses attach admission; reject before unbinding TUI ownership. + if (!adapterSupportsCreateIfDeclared(deps.adapter, record.location, record.provider)) { + throw new Error('structured_agent_session_unsupported') + } + const priorSink = host.eventSinks.eventSinkFor(input.sessionId) + const priorBarrier = await priorSink.drained() + if (!priorBarrier.ok) { + throw priorBarrier.error + } + priorSink.unbind() + // The new child writes through a sink this attempt owns; only a proven owner makes it the + // session's, so a failed acquire takes whatever its child queued with it. + const eventSink = host.eventSinks.mintEventSink(input.sessionId) + let adopted = false + try { + return await proveNativeHandoffOwner( + deps, + host, + { ...input, session, record }, + eventSink, + () => { + host.eventSinks.adoptEventSink(input.sessionId, eventSink) + adopted = true + } + ) + } finally { + if (!adopted) { + eventSink.close() + } + } +} + +async function proveNativeHandoffOwner( + deps: StructuredAgentSessionHostDeps, + host: HostHandoffAccess, + input: { + sessionId: string + fence: number + spawnToken: string + session: StructuredAgentSessionHostSession + record: AgentSessionRecord + }, + eventSink: DeferredStructuredAgentSessionEventSink, + adopt: () => void +): Promise { + const { session, record } = input + // Recheck immediately before acquisition; capability probes may drift while + // the old TUI event sink is draining. + if (!adapterSupportsCreateIfDeclared(deps.adapter, record.location, record.provider)) { + throw new Error('structured_agent_session_unsupported') + } + const acquired = await deps.adapter.acquire({ + identity: journalIdentityFor(record, session.params), + fence: input.fence, + spawnToken: input.spawnToken, + ...(record.options ? { options: record.options } : {}), + events: eventSink.sink + }) + let proved: AgentSessionRecord + try { + // A starting child has proven nothing yet: the record keeps the saved options as intent, and + // the `started` event persists what the child reports. + const options = + acquired.providerChildPhase === 'starting' + ? undefined + : await readNativeSessionOptions({ + adapter: deps.adapter, + sessionId: input.sessionId, + fence: input.fence, + ...(record.options ? { priorOptions: record.options } : {}) + }) + await deps.store.commitProcessIdentity({ + sessionId: input.sessionId, + fence: input.fence, + process: acquired.process, + now: host.now() + }) + proved = await deps.store.proveOwner({ + sessionId: input.sessionId, + fence: input.fence, + link: acquired.link, + now: host.now(), + ...(options ? { options } : {}) + }) + } catch (error) { + return rethrowAfterAgentSessionAcquisitionCleanup(deps.adapter, input.sessionId, error) + } + session.hasProviderChild = true + session.providerChildPhase = acquired.providerChildPhase ?? 'ready' + host.publishStatus?.(input.sessionId) + session.fence = proved.lease.runtimeFence + // A handoff moved the fence, not a restart: no writer is rebased across it. + delete session.resumedFromFence + session.acquisitionGeneration = acquired.acquisitionGeneration ?? null + adopt() + eventSink.bind({ + journal: session.journal, + fence: proved.lease.runtimeFence, + publish: (activity) => host.subscribers.publish(input.sessionId, session.journal, activity) + }) + const acquiredBarrier = await eventSink.drained() + if (!acquiredBarrier.ok) { + throw acquiredBarrier.error + } + host.subscribers.snapshot(input.sessionId, session.journal, proved.lease.runtimeFence) + return proved +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-option-restoration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-option-restoration.ts index 98db575a3374..d3461a7e9ee3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-option-restoration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-option-restoration.ts @@ -1,3 +1,4 @@ +import type { AgentSessionOptionsResult } from '../../../shared/agent-session-wire' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { encodeStructuredAgentSessionOptionValue } from '../../../shared/structured-agent-session-option-codec' @@ -12,22 +13,36 @@ export async function readNativeSessionOptions(input: { if (!reported) { return undefined } - const skipped = new Set(input.adapter.readOptionRestoreFailures?.(sessionId) ?? []) + return nativeSessionOptionsFromReport({ + reported: reported.current, + restoreSkipped: adapter.readOptionRestoreFailures?.(sessionId) ?? [], + ...(priorOptions ? { priorOptions } : {}) + }) +} + +/** The record's options once the provider has reported: its model, effort and Fast replace the + * saved ones, other saved options stay, and any the restore could not apply are dropped. */ +export function nativeSessionOptionsFromReport(input: { + reported: AgentSessionOptionsResult['current'] + restoreSkipped: readonly string[] + priorOptions?: Readonly> +}): Readonly> { + const { reported, priorOptions } = input const restored = priorOptions ? { ...priorOptions } : {} delete restored.model delete restored.effort delete restored.fastMode - for (const key of skipped) { + for (const key of input.restoreSkipped) { delete restored[key] } const fastMode = - reported.current.fastMode === undefined + reported.fastMode === undefined ? undefined - : encodeStructuredAgentSessionOptionValue('fastMode', reported.current.fastMode) + : encodeStructuredAgentSessionOptionValue('fastMode', reported.fastMode) return { ...restored, - model: reported.current.model, - ...(reported.current.effort ? { effort: reported.current.effort } : {}), + model: reported.model, + ...(reported.effort ? { effort: reported.effort } : {}), ...(fastMode !== undefined && fastMode !== null ? { fastMode } : {}) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts new file mode 100644 index 000000000000..52699fd062f6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts @@ -0,0 +1,147 @@ +// A publish-first create proves nothing about the model until Claude answers startup. The record +// must never hold the catalog's default in the meantime: an owner handoff or a reopen would +// replay it as a `set_model` and silently move a user whose CLI default is not Sonnet. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { AgentSessionStatusEvent } from '../../../shared/agent-session-wire' +import { ClaudeStructuredSessionAdapter } from '../../claude/claude-structured-session-adapter' +import { + fakeClaude, + PROVIDER_SESSION_ID +} from '../../claude/claude-structured-session-test-support' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +const INIT_DELAY_MS = 40 + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let adapter: ClaudeStructuredSessionAdapter +let lifecycle: Promise[] +let statuses: AgentSessionStatusEvent[] + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-provider-started-')) + resetHostTestOperationIds() + lifecycle = [] + statuses = [] + const claude = fakeClaude({ initDelayMs: INIT_DELAY_MS, initModel: 'claude-opus-9' }) + adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: root, + claudeConfigDir: join(root, 'claude-home'), + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + // A session that already minted its provider handle resumes it, as the real launch does. + resumesTranscript: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0, + continuesChain: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0 + }), + // The runtime's own mapping, so this test drives the same lifecycle path production does. + onEvent: (event) => { + const mapped = structuredClaudeLifecycleEvent(event) + if (mapped) { + lifecycle.push(host.handleAdapterEvent(mapped)) + } + }, + openConnection: claude.openConnection, + readProcessStartTime: async () => 1_700_000_000_000, + now: () => NOW + }) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + // The production router is what declares create support; the bare adapter only knows locations. + adapter: Object.assign(adapter, { supportsCreate: () => true }), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => NOW + }) + host.subscribeStatus({ id: 'status-1', emit: (event) => statuses.push(event) }) +}) + +afterEach(async () => { + await adapter.closeAll() + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function claudeParams() { + return hostTestAttachParams(null, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION_ID, leafUuid: null } + }) +} + +function lastPhase(): string | undefined { + const last = statuses.findLast((event) => event.type === 'status') + return last?.type === 'status' ? last.session.hostExecutionPhase : undefined +} + +describe('a publish-first Claude create whose init is slow', () => { + it('never persists the catalog default, and persists the reported model once started', async () => { + await expect(host.attach(CALLER, claudeParams())).resolves.toMatchObject({ ok: true }) + + // Published, not yet answering: the record holds no model rather than a guessed one. + expect(store.getRecord(SESSION)?.options?.model).toBeUndefined() + expect(lastPhase()).toBe('starting') + + await adapter.drainStartup(SESSION) + await Promise.all(lifecycle) + + expect(store.getRecord(SESSION)?.options?.model).toBe('claude-opus-9') + expect(lastPhase()).toBe('ready') + }) + + it('keeps the saved model as intent while starting, then confirms what the child runs', async () => { + const params = claudeParams() + await expect( + host.attach(CALLER, { ...params, options: { model: 'opus' } }) + ).resolves.toMatchObject({ ok: true }) + expect(store.getRecord(SESSION)?.options?.model).toBe('opus') + + await adapter.drainStartup(SESSION) + await Promise.all(lifecycle) + + expect(store.getRecord(SESSION)?.options?.model).toBe('opus') + expect(lastPhase()).toBe('ready') + }) + + it('keeps the picked model across a resume whose new child starts on its own default', async () => { + const params = claudeParams() + await host.attach(CALLER, { ...params, options: { model: 'opus' } }) + await adapter.drainStartup(SESSION) + await Promise.all(lifecycle) + await host.close(SESSION) + const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + + // Reopening the chat: the surface's first hold resumes the session. + await host.hold(SESSION, 'chat-1') + expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(releasedFence) + // The new child's init reports its CLI default; the saved pick is restored over it. + expect(store.getRecord(SESSION)?.options?.model).toBe('opus') + expect(lastPhase()).toBe('starting') + + await adapter.drainStartup(SESSION) + await Promise.all(lifecycle) + + expect(store.getRecord(SESSION)?.options?.model).toBe('opus') + expect(lastPhase()).toBe('ready') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts new file mode 100644 index 000000000000..55c718f967b9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -0,0 +1,81 @@ +// The host's half of a provider child proving its start. +// +// A publish-first acquire hands the host a child that has answered nothing yet, so the record +// keeps only the saved options the reservation carried. This is where the host learns the start +// landed, flips the session to `ready`, and persists what the child now reports as fact through +// the same record write a user's option change takes. Bookkeeping never gates the user: a failed +// write is reported and the session stays usable. +// +// This runs under the session's own serialized step, which its close and sends wait on, so it +// asks the provider nothing: the event carries what the child proved. + +import { agentSessionLeaseAdmitsWriter } from '../../../shared/agent-session-lease-adjudication' +import type { StructuredAgentSessionStartedEvent } from './structured-agent-session-adapter' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' +import { nativeSessionOptionsFromReport } from './structured-agent-session-option-restoration' + +export type StructuredAgentSessionProviderStartedContext = { + deps: StructuredAgentSessionHostDeps + sessions: Map + serialize: (sessionId: string, task: () => Promise) => Promise + now: () => number + publishStatus?: (sessionId: string) => void + restartReleaseGrace: (sessionId: string) => void + onBarrierError: (sessionId: string, error: unknown) => void +} + +export function settleStructuredAgentSessionProviderStarted( + context: StructuredAgentSessionProviderStartedContext, + event: StructuredAgentSessionStartedEvent +): Promise { + // Serialized behind the attach that published this child, so the lease it proved is committed. + return context.serialize(event.sessionId, async () => { + const session = context.sessions.get(event.sessionId) + if ( + !session?.hasProviderChild || + session.fence !== event.fence || + session.acquisitionGeneration !== event.acquisitionGeneration + ) { + return + } + session.providerChildPhase = 'ready' + // Prompts held for the start are written now but open a turn only on their echo; a release + // tick in between would stop the child before it runs them. + context.restartReleaseGrace(event.sessionId) + try { + await persistStartedOptions(context, event) + } catch (error) { + context.onBarrierError(event.sessionId, error) + } finally { + context.publishStatus?.(event.sessionId) + } + }) +} + +async function persistStartedOptions( + context: StructuredAgentSessionProviderStartedContext, + event: StructuredAgentSessionStartedEvent +): Promise { + const { store } = context.deps + const record = store.getRecord(event.sessionId) + if ( + !record || + record.lease.runtimeFence !== event.fence || + !agentSessionLeaseAdmitsWriter(record.lease) + ) { + return + } + await store.replaceSessionOptions({ + sessionId: event.sessionId, + fence: event.fence, + options: nativeSessionOptionsFromReport({ + reported: event.reportedOptions, + restoreSkipped: event.restoreSkippedOptions, + ...(record.options ? { priorOptions: record.options } : {}) + }), + now: context.now() + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts index 34fd6452b08f..543a1a7be709 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts @@ -20,6 +20,7 @@ export type RestoredStructuredAgentSessionRead = { params: AgentSessionAttachParams fence: number hasProviderChild: false + providerChildPhase: 'ready' acquisitionGeneration: null } @@ -67,6 +68,7 @@ export async function restoreStructuredAgentSessionRead( params, fence: record.lease.runtimeFence, hasProviderChild: false, + providerChildPhase: 'ready', acquisitionGeneration: null } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts index a0bb32ad7374..a0099be32a71 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts @@ -4,6 +4,7 @@ import type { AgentSessionRecordStore } from '../../runtime/agent-session-record import type { RestoredStructuredAgentSessionRead } from './structured-agent-session-read-restore' import { restoreOneStructuredAgentSessionRead, + restoreOneStructuredAgentSessionReadUnderSerialize, restoreStructuredAgentSessionsOnRestart } from './structured-agent-session-restart-restore' @@ -48,14 +49,29 @@ export class StructuredAgentSessionReadableRestorer { * answers for Claude and Codex from the record's own provider. */ async restoreOne(sessionId: string): Promise { - const record = this.input.store.getRecord(sessionId) - if (!record || !this.input.supportsRecord(record)) { + if (!this.supports(sessionId)) { return false } await restoreOneStructuredAgentSessionRead(this.input, sessionId) return this.input.hasSession(sessionId) } + /** `restoreOne` for a caller already inside the session's serialize. Reconciliation is skipped + * on purpose: a lease this host has not adjudicated is the attach's problem, and a replay + * needs only the journal. */ + async restoreOneUnderSerialize(sessionId: string): Promise { + if (!this.supports(sessionId)) { + return false + } + await restoreOneStructuredAgentSessionReadUnderSerialize(this.input, sessionId) + return this.input.hasSession(sessionId) + } + + private supports(sessionId: string): boolean { + const record = this.input.store.getRecord(sessionId) + return record !== null && this.input.supportsRecord(record) + } + private async restoreReadableSessions(sessionIds?: readonly string[]): Promise { const targetOrder = sessionIds ? new Map(sessionIds.map((sessionId, index) => [sessionId, index])) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts index ccad5f7225f7..97f5321e8072 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts @@ -100,6 +100,8 @@ async function createHarness(options: { attached?: boolean; transport?: boolean }), cancelTurn: async () => ({ cancelled: true }), answerPrompt: async () => undefined, + // A failed acquisition is proven gone, as the real adapters prove it. + releaseAcquisition: async () => true, setOption } const host = new StructuredAgentSessionHost({ @@ -247,7 +249,9 @@ const UNREACHABLE = new Set([ 'agentSession.setOption:agent_session_journal_unreadable', 'agentSession.send:agent_session_journal_unreadable', // Send reconstructs doubt from its global tombstone instead of refusing it. - 'agentSession.send:agent_session_operation_unknown' + 'agentSession.send:agent_session_operation_unknown', + // Only a send restarts a lost owner. + 'agentSession.setOption:agent_session_owner_restart_failed' ]) describe('agentSessionRefusalOperationState host oracle', () => { @@ -373,6 +377,19 @@ describe('agentSessionRefusalOperationState host oracle', () => { ) } + const unrecoverable = await createHarness() + await unrecoverable.host.close(SESSION) + unrecoverable.host.deps.adapter.acquire = async () => { + throw new Error('no provider thread to resume') + } + record( + await assertHostAgreement( + unrecoverable, + { method: 'agentSession.send', operationId: operationId() }, + 'agent_session_owner_restart_failed' + ) + ) + const allPairs = METHODS.flatMap((method) => AGENT_SESSION_WIRE_REFUSAL_CODES.map((code) => `${method}:${code}` as Pair) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-release-clock.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-release-clock.ts index 452e6a6fae03..1cce910c09d6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-release-clock.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-release-clock.ts @@ -1,20 +1,22 @@ -// The delay between "nothing holds this session" and "stop its provider child". +// The delay between "nothing holds this session and nothing has happened in it" and "stop its +// provider child". // -// TWO reasons it is not immediate. A surface that reconnects — a mobile socket dropping on a -// network switch, a renderer remounting a tab — releases and re-holds within a second, and killing -// an app-server in that window costs the user a respawn plus a resume for nothing. And a turn the -// user already asked for must finish: the provider is mid-answer, the journal has an open turn -// marker, and stopping the child there strands both. +// It is an IDLE window, not a short grace. A surface that reconnects — a mobile socket dropping on +// a network switch, a renderer remounting a tab, a worktree switch hiding the pane — releases and +// re-holds, and a send to a chat nobody is looking at restarts its owner; stopping the child soon +// after either costs the user a respawn plus a resume on the next message. And a turn the user +// already asked for must finish: stopping the child mid-answer strands the open turn marker. // -// So the clock arms when the last holder leaves, and a tick that finds a turn still running RE-ARMS -// instead of evicting. That is what makes the wait start at the later of the two events rather than -// at whichever came first. +// So the clock arms when the last holder leaves, every journal write while it is armed starts it +// again, and a tick that finds work still owed — a turn running, or a message sent but not yet +// taken by the provider — re-arms instead of evicting. The child goes only after a full window +// with no holder and no owed work. Quit still stops every child at once. -export const STRUCTURED_AGENT_SESSION_RELEASE_GRACE_MS = 15_000 +export const STRUCTURED_AGENT_SESSION_RELEASE_GRACE_MS = 30 * 60_000 export type StructuredAgentSessionReleaseClockDeps = { - /** Never evict mid-turn; a true answer re-arms the clock instead. */ - isTurnActive: (sessionId: string) => boolean + /** Never evict while work is owed; a true answer re-arms the clock instead. */ + hasOwedWork: (sessionId: string) => boolean /** Re-checked at fire time: a holder may have arrived while the timer ran. */ isHeld: (sessionId: string) => boolean evict: (sessionId: string) => Promise @@ -41,6 +43,13 @@ export class StructuredAgentSessionReleaseClock { this.timers.set(sessionId, timer) } + /** Activity in an unheld session: the idle window starts over. */ + renew(sessionId: string): void { + if (this.timers.has(sessionId)) { + this.arm(sessionId) + } + } + cancel(sessionId: string): void { const timer = this.timers.get(sessionId) if (timer) { @@ -64,7 +73,7 @@ export class StructuredAgentSessionReleaseClock { if (this.deps.isHeld(sessionId)) { return } - if (this.deps.isTurnActive(sessionId)) { + if (this.deps.hasOwedWork(sessionId)) { this.arm(sessionId) return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 7e697efbdc73..8cc202b28c76 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -53,24 +53,36 @@ export async function restoreOneStructuredAgentSessionRead( // A session latched in recovery exits here at startup, without waiting for a client. await input.resolveRecovery(sessionId) } - await input.serialize(sessionId, async () => { - if (input.hasSession(sessionId)) { - // A surface that took a hold mid-restore already attached this one. - await input.restoreHandoff(sessionId) - return - } - const restored = await restoreStructuredAgentSessionRead( - input.store, - input.journalRoot, - sessionId - ) - if (!restored) { - return - } - input.onReadable(sessionId, restored) - await input.retrySettlement(sessionId, restored.params) + await input.serialize(sessionId, () => + restoreOneStructuredAgentSessionReadUnderSerialize(input, sessionId) + ) +} + +/** The serialized half of the restore, for a caller already inside the session's serialize — a + * send replaying into a session this host has closed, which needs the journal and no child. */ +export async function restoreOneStructuredAgentSessionReadUnderSerialize( + input: Pick< + StructuredAgentSessionReadRestoreDeps, + 'store' | 'journalRoot' | 'hasSession' | 'onReadable' | 'retrySettlement' | 'restoreHandoff' + >, + sessionId: string +): Promise { + if (input.hasSession(sessionId)) { + // A surface that took a hold mid-restore already attached this one. await input.restoreHandoff(sessionId) - }) + return + } + const restored = await restoreStructuredAgentSessionRead( + input.store, + input.journalRoot, + sessionId + ) + if (!restored) { + return + } + input.onReadable(sessionId, restored) + await input.retrySettlement(sessionId, restored.params) + await input.restoreHandoff(sessionId) } export async function restoreStructuredAgentSessionsOnRestart( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index d940fb323bd1..8676e86acfbb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -66,6 +66,8 @@ export function createStructuredAgentSessionHostRestore( ): { restoreReadableSessions: (sessionIds?: readonly string[]) => Promise revealSession: (sessionId: string) => Promise + /** One session, for a caller already inside its serialize. */ + restoreReadableUnderSerialize: (sessionId: string) => Promise } { const restorer = new StructuredAgentSessionReadableRestorer({ store: deps.store, @@ -81,6 +83,7 @@ export function createStructuredAgentSessionHostRestore( revealSession: (sessionId) => revealStructuredAgentSession(deps, sessionId, wiring.hasSession, (id) => restorer.restoreOne(id) - ) + ), + restoreReadableUnderSerialize: (sessionId) => restorer.restoreOneUnderSerialize(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index f6372c3d8ed7..baedbe7c4d19 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -35,7 +35,7 @@ export async function rewindStructuredAgentSession( adapter: context.deps.adapter, callerKey: caller.callerKey, envelope: params.envelope, - journal: context.sessions.get(sessionId)?.journal, + journal: () => context.sessions.get(sessionId)?.journal, publish: (journal) => context.publish(sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, now: context.now, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts new file mode 100644 index 000000000000..a467a318c808 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts @@ -0,0 +1,678 @@ +// A send, or a hold, that finds the session's provider child gone, against the real host. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { agentSessionRefusalOperationState } from '../../../shared/agent-session-refusal-retry' +import type { AgentSessionMutationEnvelope } from '../../../shared/agent-session-wire' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +// Long enough that no release fires mid-test; whether one is pending is asserted directly. +const GRACE_MS = 60_000 + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let spawnChild: StructuredAgentSessionAdapter['acquire'] +let dispatch: Mock +let hostErrors: unknown[] + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-send-recovery-')) + resetHostTestOperationIds() + hostErrors = [] + let generation = 0 + spawnChild = async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + acquisitionGeneration: `generation-${++generation}`, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } + }) + acquire = vi.fn(spawnChild) + dispatch = vi.fn(async () => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD, + turnId: `turn-${dispatch.mock.calls.length}`, + ordinal: dispatch.mock.calls.length + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: vi.fn(async () => ({ cancelled: false })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined) + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + releaseGraceMs: GRACE_MS, + now: () => NOW, + onEventSinkError: ({ error }) => hostErrors.push(error) + }) + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function sendParams(text: string, operationId = hostTestOperationId()) { + const body = hostTestMessage(text) + const envelope: AgentSessionMutationEnvelope = { + sessionId: SESSION, + clientOperationId: operationId, + expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + } + return { envelope, body } +} + +/** Every status row the chat shows, oldest first; none when the session is not even readable. */ +function journalStatuses(): string[] { + if (!host.hasSession(SESSION)) { + return [] + } + const history = host.history({ sessionId: SESSION, direction: 'tail' }) + return history.ok + ? history.page.items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) + : [] +} + +/** The child timed out or exited: its lease is handed back and the host holds no session. */ +async function loseOwner(): Promise { + await host.close(SESSION) + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'released', + ownerProcess: null + }) + acquire.mockClear() +} + +describe('a send with no live owner', () => { + it('restarts the owner once and delivers against it', async () => { + await loseOwner() + + const result = await host.send(CALLER, sendParams('after the child died')) + + expect(result).toMatchObject({ ok: true, replayed: false }) + expect(acquire).toHaveBeenCalledOnce() + expect(dispatch).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + }) + + it('restarts the owner before the send is admitted, so the send is admitted once', async () => { + await loseOwner() + const order: string[] = [] + const spawnChild = acquire.getMockImplementation()! + acquire.mockImplementationOnce(async (input) => { + order.push('acquire') + return spawnChild(input) + }) + const admit = store.admitMutationOperation + vi.spyOn(store, 'admitMutationOperation').mockImplementation((args) => { + order.push('admit') + return admit(args) + }) + + await expect(host.send(CALLER, sendParams('ensure first'))).resolves.toMatchObject({ + ok: true, + replayed: false + }) + + expect(order).toEqual(['acquire', 'admit']) + }) + + it('leaves a live owner alone', async () => { + acquire.mockClear() + + await expect(host.send(CALLER, sendParams('owner is live'))).resolves.toMatchObject({ + ok: true + }) + + expect(acquire).not.toHaveBeenCalled() + }) + + it('does not restart an owner for a send the session refuses anyway', async () => { + await loseOwner() + await store.transitionHandoff(SESSION, (current) => ({ + ...current, + conversationCommand: { + command: 'clear', + state: 'completed', + replacementSessionId: 'session-after-clear', + operationId: hostTestOperationId(), + callerKey: CALLER.callerKey, + phase: 'committed' + } + })) + + await host.send(CALLER, sendParams('into a cleared chat')) + + expect(acquire).not.toHaveBeenCalled() + }) + + it('renews the idle window on journal activity in an unheld session', async () => { + await loseOwner() + await expect(host.send(CALLER, sendParams('restart'))).resolves.toMatchObject({ ok: true }) + const arm = vi.spyOn(host['holds']['clock'], 'arm') + + await expect(host.send(CALLER, sendParams('more activity'))).resolves.toMatchObject({ + ok: true + }) + + expect(arm).toHaveBeenCalledWith(SESSION) + }) + + it('releases the restarted child on the usual clock only when no surface holds it', async () => { + await loseOwner() + await expect(host.send(CALLER, sendParams('nobody is watching'))).resolves.toMatchObject({ + ok: true + }) + expect(host['holds'].isReleasePending(SESSION)).toBe(true) + + await host.close(SESSION) + // A reading surface that does not itself restart the agent. + await host.hold(SESSION, 'desktop-chat:1', { resume: false }) + await expect(host.send(CALLER, sendParams('the chat is open'))).resolves.toMatchObject({ + ok: true + }) + expect(host['holds'].isReleasePending(SESSION)).toBe(false) + }) + + it('restarts an owner that exited while the session stayed readable', async () => { + const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + await host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'provider exited', + cause: 'unexpected-exit', + fence, + acquisitionGeneration: 'generation-1' + }) + expect(host.hasSession(SESSION)).toBe(true) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + acquire.mockClear() + + await expect(host.send(CALLER, sendParams('after an exit'))).resolves.toMatchObject({ + ok: true + }) + expect(acquire).toHaveBeenCalledOnce() + }) + + it('restarts nothing for a resend the journal already answers', async () => { + const params = sendParams('sent once') + await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + // The child died during startup: the lease is handed back, the fence moves, and the session + // stays readable. The client resends against the new fence. + await host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'startup deadline', + cause: 'unexpected-exit', + fence: params.envelope.expectedRuntimeFence ?? 0, + acquisitionGeneration: 'generation-1', + startupUnproven: true + }) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + acquire.mockClear() + const resent = { + ...params, + envelope: { + ...params.envelope, + expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + } + } + + await expect(host.send(CALLER, resent)).resolves.toMatchObject({ ok: true, replayed: true }) + + expect(acquire).not.toHaveBeenCalled() + expect(dispatch).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + + // Retry rotates the id: a genuinely new send restarts the owner once. + await expect(host.send(CALLER, sendParams('sent once'))).resolves.toMatchObject({ + ok: true, + replayed: false + }) + expect(acquire).toHaveBeenCalledOnce() + expect(dispatch).toHaveBeenCalledTimes(2) + }) + + it('restarts nothing for a send the ledger holds but the journal never saw', async () => { + const params = sendParams('claimed, then the host died') + // The row was claimed and the host went down before the journal write: on replay, admission + // reconstructs an unknown-outcome submission and never needs an owner. + await store.admitMutationOperation({ + callerKey: CALLER.callerKey, + envelope: params.envelope, + hostFingerprint: params.envelope.payloadFingerprint, + now: NOW, + operationIdScope: 'global' + }) + await store.recordOperationOutcome({ + callerKey: CALLER.callerKey, + operationId: params.envelope.clientOperationId, + outcome: { status: 'unknown' } + }) + await host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'provider exited', + cause: 'unexpected-exit', + fence: params.envelope.expectedRuntimeFence ?? 0, + acquisitionGeneration: 'generation-1' + }) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + acquire.mockClear() + + const result = await host.send(CALLER, { + ...params, + envelope: { + ...params.envelope, + expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + } + }) + + expect(result).toMatchObject({ + ok: true, + replayed: true, + value: { submission: { dispatchState: 'unknown', recovered: true } } + }) + expect(acquire).not.toHaveBeenCalled() + expect(dispatch).not.toHaveBeenCalled() + }) + + it('rebases a send that arrives after the restart has already claimed the lease', async () => { + await loseOwner() + const lostFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + let claimed = () => {} + let release = () => {} + const claim = new Promise((resolve) => (claimed = resolve)) + const spawn = new Promise((resolve) => (release = resolve)) + const spawnChild = acquire.getMockImplementation() + acquire.mockImplementationOnce(async (input) => { + claimed() + await spawn + return spawnChild!(input) + }) + + const first = host.send(CALLER, sendParams('first')) + await claim + expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(lostFence + 1) + const late = sendParams('second') + late.envelope.expectedRuntimeFence = lostFence + const second = host.send(CALLER, late) + release() + + expect(await first).toMatchObject({ ok: true }) + expect(await second).toMatchObject({ ok: true }) + expect(acquire).toHaveBeenCalledOnce() + expect(dispatch).toHaveBeenCalledTimes(2) + }) + + it('shares one restart between concurrent sends', async () => { + await loseOwner() + + const results = await Promise.all([ + host.send(CALLER, sendParams('first')), + host.send(CALLER, sendParams('second')), + host.send(CALLER, sendParams('third')) + ]) + + expect(results.map((result) => result.ok)).toEqual([true, true, true]) + expect(acquire).toHaveBeenCalledOnce() + expect(dispatch).toHaveBeenCalledTimes(3) + }) + + it('shares one restart between a hold and a send that arrive in the same gap', async () => { + await loseOwner() + + const [held, sent] = await Promise.allSettled([ + host.hold(SESSION, 'desktop-chat:1'), + host.send(CALLER, sendParams('while the chat opens')) + ]) + + expect(held).toMatchObject({ status: 'fulfilled' }) + expect(sent).toMatchObject({ status: 'fulfilled', value: { ok: true } }) + expect(acquire).toHaveBeenCalledOnce() + expect(dispatch).toHaveBeenCalledOnce() + expect(host['holds'].isHeld(SESSION)).toBe(true) + expect(host['holds'].isReleasePending(SESSION)).toBe(false) + }) + + it('replays into a closed session without spawning anything', async () => { + const params = sendParams('sent once') + await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + await loseOwner() + + await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) + await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) + + // The journal was made readable for the answer; the record's lease was left as it was. + expect(host.hasSession(SESSION)).toBe(true) + expect(acquire).not.toHaveBeenCalled() + expect(dispatch).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + expect(host['holds'].isReleasePending(SESSION)).toBe(false) + }) + + it("refuses with the restart's own cause, in the answer and in the chat", async () => { + await loseOwner() + acquire.mockRejectedValue(new Error('Not signed in. Run codex login')) + const params = sendParams('while signed out') + + const result = await host.send(CALLER, params) + + expect(result).toEqual({ + ok: false, + refusal: { + code: 'agent_session_owner_restart_failed', + message: "Codex couldn't restart: Not signed in. Run codex login.", + // The failed attach proved its child gone: nothing runs for this session. + ownerVerdict: 'exited' + } + }) + expect(dispatch).not.toHaveBeenCalled() + expect(hostErrors).not.toEqual([]) + expect( + agentSessionRefusalOperationState('agentSession.send', 'agent_session_owner_restart_failed') + ).toBe('settled-rejected') + // Refused before admission: the ledger holds nothing a resend would replay. + expect(store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId)).toBeNull() + // The same status row a failed start leaves, so the reason outlives the error strip. + expect(journalStatuses()).toEqual([ + 'The provider stopped before it finished starting: Not signed in. Run codex login.' + ]) + }) + + it('restarts again for a Retry of the refused send, under its own id or a new one', async () => { + await loseOwner() + acquire.mockRejectedValue(new Error('Not signed in')) + const params = sendParams('while signed out') + await expect(host.send(CALLER, params)).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_owner_restart_failed' } + }) + + // A client that resends the same id gets another attempt, and the chat no second row. + await expect(host.send(CALLER, params)).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_owner_restart_failed' } + }) + expect(acquire).toHaveBeenCalledTimes(2) + expect(journalStatuses()).toHaveLength(1) + + // The outbox's Retry rotates the id: also a fresh attempt. + await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_owner_restart_failed' } + }) + expect(acquire).toHaveBeenCalledTimes(3) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('restarts and delivers a later send once the cause clears', async () => { + await loseOwner() + acquire.mockRejectedValueOnce(new Error('Not signed in')) + await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_owner_restart_failed' } + }) + + // The user signed in; nothing about the failed attempt is remembered. + await expect(host.send(CALLER, sendParams('signed in now'))).resolves.toMatchObject({ + ok: true, + replayed: false + }) + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + }) + + it('suggests a new chat only when this host has nothing to restart the chat from', async () => { + await loseOwner() + acquire.mockRejectedValue(new Error('Not signed in')) + const failed = await host.send(CALLER, sendParams('restart fails')) + expect(failed).toMatchObject({ + ok: false, + refusal: { code: 'agent_session_owner_restart_failed' } + }) + expect(failed.ok ? '' : failed.refusal.message).not.toMatch(/new chat/) + + // The adapter cannot run this record where it lives: no retry would bring it back. + host.deps.adapter.supportsLocation = () => false + const unresumable = await host.send(CALLER, sendParams('cannot resume here')) + + expect(unresumable).toMatchObject({ + ok: false, + refusal: { + code: 'agent_session_owner_restart_failed', + message: + "Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue." + } + }) + }) + + it('runs the send as the lease stands when the restart met a lease someone else is settling', async () => { + await loseOwner() + vi.spyOn(host['holds'], 'ensureProviderChild').mockResolvedValueOnce({ + ok: false, + refusal: { + code: 'execution_owner_reconciling', + message: 'Another runtime is still adjudicating this lease.' + } + }) + + const result = await host.send(CALLER, sendParams('owner being settled')) + + // The ordinary lease check answers, retryably; nothing terminal and nothing in the chat. + expect(result).toMatchObject({ + ok: false, + refusal: { code: 'agent_session_ownership_unknown' } + }) + expect(acquire).not.toHaveBeenCalled() + expect(journalStatuses()).toEqual([]) + }) + + it('runs the send as the lease stands when the restart itself faults', async () => { + await loseOwner() + vi.spyOn(host['holds'], 'ensureProviderChild').mockRejectedValueOnce( + new Error('spawn-token mint failed') + ) + + const result = await host.send(CALLER, sendParams('bookkeeping failed')) + + expect(result).toMatchObject({ + ok: false, + refusal: { code: 'agent_session_ownership_unknown' } + }) + expect(hostErrors).toContainEqual( + expect.objectContaining({ message: 'spawn-token mint failed' }) + ) + expect(journalStatuses()).toEqual([]) + }) + + it('keeps a second surface holder taken during an auto-restart, and starts nothing for it', async () => { + await host.hold(SESSION, 'desktop-chat:1') + const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + acquire.mockClear() + const entered = Promise.withResolvers() + const gate = Promise.withResolvers() + const spawnChild = acquire.getMockImplementation()! + acquire.mockImplementationOnce(async (input) => { + entered.resolve() + await gate.promise + return spawnChild(input) + }) + + // The held child exits: the host restarts it on its own, under the surface that holds it. + const restarted = host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'provider exited', + cause: 'unexpected-exit', + fence: exitedFence, + acquisitionGeneration: 'generation-1' + }) + await entered.promise + const second = host.hold(SESSION, 'paired-phone:1') + gate.resolve() + await Promise.all([restarted, second]) + + expect(acquire).toHaveBeenCalledOnce() + expect(host['holds']['holders'].holderIds(SESSION)).toEqual([ + 'desktop-chat:1', + 'paired-phone:1' + ]) + expect(host['holds'].isReleasePending(SESSION)).toBe(false) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + }) + + it('puts the child of an auto-restart on the idle clock when its surface left mid-attach', async () => { + await host.hold(SESSION, 'desktop-chat:1') + const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + acquire.mockClear() + const entered = Promise.withResolvers() + const gate = Promise.withResolvers() + const spawnChild = acquire.getMockImplementation()! + acquire.mockImplementationOnce(async (input) => { + entered.resolve() + await gate.promise + return spawnChild(input) + }) + + const restarted = host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'provider exited', + cause: 'unexpected-exit', + fence: exitedFence, + acquisitionGeneration: 'generation-1' + }) + await entered.promise + // The only surface leaves while the host is still spawning the child it asked for. + host.release(SESSION, 'desktop-chat:1') + gate.resolve() + await restarted + + expect(acquire).toHaveBeenCalledOnce() + expect(host['holds'].isHeld(SESSION)).toBe(false) + // Nobody holds the child, so it goes on the same clock a departed surface would start. + expect(host['holds'].isReleasePending(SESSION)).toBe(true) + expect(hostErrors).toEqual([]) + }) + + it('counts a queued restart as in flight from the moment it is asked for, so a quit drains it', async () => { + const closeSession = vi.mocked(host.deps.adapter.closeSession!) + acquire.mockClear() + const gate = Promise.withResolvers() + closeSession.mockImplementationOnce(async () => { + await gate.promise + return true + }) + const closing = host.close(SESSION) + const hold = host.hold(SESSION, 'desktop-chat:1') + let drained = false + void host['tasks'].drainAttaches().then(() => { + drained = true + }) + await new Promise((resolve) => setImmediate(resolve)) + + // The hold waits its turn behind the close, and the quit's drain waits for the hold: the + // child it is about to spawn must exist before the quit decides what to evict. + expect(acquire).not.toHaveBeenCalled() + expect(drained).toBe(false) + + gate.resolve() + await Promise.all([closing, hold]) + await new Promise((resolve) => setImmediate(resolve)) + expect(acquire).toHaveBeenCalledOnce() + expect(drained).toBe(true) + }) + + it('adjudicates a lease this host has not reconciled before a hold resumes it', async () => { + await loseOwner() + await store.transitionHandoff(SESSION, (current) => ({ + ...current, + lease: { ...current.lease, unreconciled: true } + })) + host.deps.probeOwner = async () => ({ outcome: 'pid-absent' }) + + await host.hold(SESSION, 'desktop-chat:1') + + expect(acquire).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + unreconciled: false, + claimStatus: 'live' + }) + }) + + it('exits the recovery stage a failed attempt latched before a hold resumes', async () => { + await loseOwner() + // What an acquisition whose exit could not be proven leaves behind: nobody's, but latched. + await store.transitionHandoff(SESSION, (current) => ({ + ...current, + lease: { ...current.lease, handoffStage: 'manual-recovery' } + })) + host.deps.probeOwner = async () => ({ outcome: 'pid-absent' }) + + await host.hold(SESSION, 'desktop-chat:1') + + expect(acquire).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + handoffStage: null, + claimStatus: 'live' + }) + }) + + it('leaves a lease it cannot adjudicate alone', async () => { + await loseOwner() + await store.transitionHandoff(SESSION, (current) => ({ + ...current, + lease: { ...current.lease, unreconciled: true } + })) + + const result = await host.send(CALLER, sendParams('owner unverifiable')) + + expect(result).toMatchObject({ + ok: false, + refusal: { code: 'agent_session_ownership_unknown' } + }) + expect(acquire).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts new file mode 100644 index 000000000000..e4a332f7acf8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts @@ -0,0 +1,244 @@ +// What a send needs from the session before its lease is checked. +// +// A provider child that exits or fails to start hands its lease back. Before this, a send to that +// session was refused `agent_session_ownership_unknown` — which a client reads as "not admitted +// yet" and resends forever — and only a surface hold could ever make a new child. Now the send +// makes sure it has an owner as a step of its own serialized admission: a released lease where +// resume is allowed gets a child first; anything else runs as it is and meets the lease check. +// A restart that fails refuses with a code the client stops auto-retrying on, carrying the +// restart's own cause, and writes that cause into the chat the way a start that failed does, so +// the user sees why. A manual Retry or a new send is a fresh attempt: a refusal before admission +// leaves no ledger row behind. +// +// The ledger's answer comes first, so a send it already holds a row for restarts nothing: +// admission replays or refuses it whoever owns the session now, and a closed session is made +// readable for that, never given a child. Otherwise a child that dies at startup moves the fence, +// the client resends the same message against the new fence, and each replay spawns another +// child that dies the same way. +// +// Running inside the send's serialize is what makes "no child" exact and the fence bookkeeping +// simple: the owner this send (or a hold just ahead of it) replaced is the one the client was +// current as of, so the send is admitted at the fence the restart published. +// +// A child that has not proven its start is still the owner: the send is admitted against it and +// the adapter holds the message until startup lands, or rejects it with the child's own reason +// when the child dies first. The exit settlement writes that reason into the chat. + +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { + AgentSessionMutationEnvelope, + AgentSessionWireRefusal +} from '../../../shared/agent-session-wire' +import type { AgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals' +import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds' +import { TUI_AGENT_DISPLAY_NAMES } from '../../../shared/tui-agent-display-names' +import { + ownerRestartFailedOutcome, + providerStartupFailureOutcome +} from './structured-agent-session-dead-generation-settlement' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' +import type { AgentSessionMutationSessionPreparation } from './structured-agent-session-mutation-admission' +import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' +import { rewindRefusal } from './structured-rewind-refusal' + +/** + * What a refused resume means for the send that ran it. `transient`: the resume met a lease + * someone else is settling, which is not proof it cannot resume — the send runs as the lease + * stands and admission reports it. `failed`: the restart itself failed; the send answers with the + * cause and stops the client's retry loop, and the user may clear the cause and retry. + * `unresumable`: this host has nothing to restart the chat from — no record, or none it can run — + * so only a new chat continues. A new wire code does not compile until it is classified here. + */ +const RESUME_REFUSAL_OUTCOME: Record< + AgentSessionWireRefusalCode, + 'transient' | 'failed' | 'unresumable' +> = { + execution_owner_reconciling: 'transient', + agent_session_conflict: 'transient', + agent_session_checkpoint_stale: 'transient', + agent_session_ownership_unknown: 'transient', + agent_session_operation_capacity: 'transient', + structured_agent_session_unsupported: 'unresumable', + agent_session_operation_conflict: 'failed', + agent_session_operation_expired: 'failed', + agent_session_operation_invalid: 'failed', + agent_session_operation_unknown: 'failed', + agent_session_item_revision_stale: 'failed', + agent_session_already_resolved: 'failed', + agent_session_identity_required: 'unresumable', + agent_session_journal_unreadable: 'failed', + agent_session_owner_restart_failed: 'failed' +} + +/** Why the record refuses any send right now, whoever owns it; null when a send may run. */ +export function structuredAgentSessionSendBlock( + record: AgentSessionRecord | null +): { ok: false; refusal: AgentSessionWireRefusal } | null { + const rewind = record?.rewind + if (rewind?.phase === 'prepared' || rewind?.phase === 'provider-succeeded') { + return rewindRefusal('outcome-unknown') + } + const command = record?.conversationCommand + if ( + command && + ((command.state === 'unknown' && command.phase === 'prepared') || + (command.command === 'clear' && command.replacementSessionId)) + ) { + return { + ok: false, + refusal: { + code: 'agent_session_operation_invalid', + message: command.replacementSessionId + ? 'This conversation has been cleared. Use the current conversation.' + : 'The conversation operation is unconfirmed.' + } + } + } + return null +} + +/** Whether this send is the one that must bring the owner back: no child, a lease handed back + * cleanly, and nothing on the record that refuses the send anyway. Live, unverifiable, still + * reserved, or handed off: that lease is not this send's to replace. */ +export function structuredAgentSessionSendNeedsOwner( + session: StructuredAgentSessionHostSession | undefined, + record: AgentSessionRecord +): boolean { + return ( + session?.hasProviderChild !== true && + isResumableStructuredAgentSessionRecord(record) && + structuredAgentSessionSendBlock(record) === null + ) +} + +type SendPreparationContext = Pick< + StructuredAgentSessionMutationContext, + 'deps' | 'sessions' | 'holds' | 'restoreReadable' | 'publish' +> + +export async function prepareStructuredAgentSessionSend( + context: SendPreparationContext, + envelope: AgentSessionMutationEnvelope, + ledger: 'admit' | 'replay', + record: AgentSessionRecord +): Promise { + const { sessionId } = record + if (ledger !== 'admit') { + if (!context.sessions.has(sessionId)) { + await context.restoreReadable(sessionId) + } + return { ok: true, envelope } + } + if (structuredAgentSessionSendNeedsOwner(context.sessions.get(sessionId), record)) { + const refusal = await restartOwnerForSend(context, envelope, record) + if (refusal) { + return { ok: false, refusal } + } + } + return { ok: true, envelope: admitAtResumedFence(context.sessions.get(sessionId), envelope) } +} + +/** One restart attempt. Answers with the refusal that ends the send, or null when the send goes + * on to admission — after a child, after a transient refusal, or after a fault in the restart's + * own bookkeeping, which is reported and never gates the user's action. */ +async function restartOwnerForSend( + context: SendPreparationContext, + envelope: AgentSessionMutationEnvelope, + record: AgentSessionRecord +): Promise { + const { sessionId } = envelope + let resumed: Awaited> + try { + resumed = await context.holds.ensureProviderChild(sessionId) + } catch (error) { + context.deps.onEventSinkError?.({ sessionId, error }) + return null + } + const outcome = resumed.ok ? null : RESUME_REFUSAL_OUTCOME[resumed.refusal.code] + if (resumed.ok || outcome === 'transient') { + return null + } + const refusal = ownerRestartFailedRefusal(record, resumed.refusal, outcome !== 'unresumable') + context.deps.onEventSinkError?.({ + sessionId, + error: new Error(`${resumed.refusal.code}: ${resumed.refusal.message}`) + }) + // A restart whose child died starting leaves the row any start that died leaves, so the chat + // reads the same whether the send met that death before admission or after it. + await recordFailedRestart( + context, + envelope, + resumed.refusal.ownerVerdict === 'exited' + ? providerStartupFailureOutcome(resumed.refusal.message) + : refusal.message + ) + return refusal +} + +/** The client stops on the code; the message carries the restart's own cause, and the verdict — + * when the failed attach proved its child gone — tells a client nothing runs for the session. */ +function ownerRestartFailedRefusal( + record: AgentSessionRecord, + cause: AgentSessionWireRefusal, + resumable: boolean +): AgentSessionWireRefusal { + return { + code: 'agent_session_owner_restart_failed', + message: ownerRestartFailedOutcome({ + agentName: TUI_AGENT_DISPLAY_NAMES[record.provider], + reason: cause.message, + resumable + }), + ...(cause.ownerVerdict ? { ownerVerdict: cause.ownerVerdict } : {}) + } +} + +/** The same status row a start that failed leaves in the chat, so the reason outlives the error + * strip. The journal is made readable for it when the failed attach left none behind. Keyed by + * the send, not the clock: a resend of the same id that fails again adds no second row. */ +async function recordFailedRestart( + context: SendPreparationContext, + envelope: AgentSessionMutationEnvelope, + text: string +): Promise { + const { sessionId } = envelope + try { + if (!context.sessions.has(sessionId)) { + await context.restoreReadable(sessionId) + } + const session = context.sessions.get(sessionId) + if (!session) { + return + } + const settlementId = `failed-restart:${envelope.clientOperationId}` + await session.journal.appendLifecycleBatch({ + settlementId, + fence: session.fence, + recovered: true, + mutations: [ + { + kind: 'item', + identity: { provider: 'orca', clientMessageId: settlementId }, + body: { kind: 'status', text: boundJournalStatusText(text) } + } + ] + }) + context.publish(sessionId, session.journal) + } catch (error) { + context.deps.onEventSinkError?.({ sessionId, error }) + } +} + +/** A writer current as of the owner this child replaced is current now: the restart was the only + * thing that moved the fence, whether this send ran it or one just ahead of it did. */ +function admitAtResumedFence( + session: StructuredAgentSessionHostSession | undefined, + envelope: AgentSessionMutationEnvelope +): AgentSessionMutationEnvelope { + return session?.hasProviderChild && + session.resumedFromFence !== undefined && + envelope.expectedRuntimeFence === session.resumedFromFence + ? { ...envelope, expectedRuntimeFence: session.fence } + : envelope +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts new file mode 100644 index 000000000000..181b40660f8f --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts @@ -0,0 +1,257 @@ +// A session that published and then lost its child before startup (not signed in, say) keeps a +// released lease and a chat the user can still type into. The send is the user asking for the +// child back: the host restarts it before admitting the write and delivers against the new owner, +// instead of parking the message behind a lease nothing would ever re-acquire. +// +// A child is published before it has proven its start, and it owns the send from that moment: the +// message is admitted against it and the adapter holds it for the start. When the child exits +// first, the exit settlement rejects the message and writes the cause into the chat, once, and +// the next send is a fresh restart. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionMutationEnvelope } from '../../../shared/agent-session-wire' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +const EXIT_REASON = 'Claude Code is not signed in. Sign in with the Claude CLI' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let dispatch: Mock +let generation = 0 + +function sendEnvelope( + fence: number, + body: ReturnType +): AgentSessionMutationEnvelope { + return { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + } +} + +/** A send is admitted against the child it meets, proven or not; the adapter holds the rest. */ +async function send( + text: string, + fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 +): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { envelope: sendEnvelope(fence, body), body }) + expect(sent, JSON.stringify(sent)).toMatchObject({ + ok: true, + replayed: false, + value: { submission: { dispatchState: 'pending' } } + }) + return sent.ok ? sent.value.clientMessageId : '' +} + +/** The child of the current acquisition, as the adapter would identify it in a lifecycle event. */ +function currentChild() { + return { + sessionId: SESSION, + fence: store.getRecord(SESSION)?.lease.runtimeFence ?? 0, + acquisitionGeneration: `generation-${generation}` + } +} + +function proveStarted(): Promise { + return host.handleAdapterEvent({ + type: 'started', + ...currentChild(), + reportedOptions: { model: 'sonnet' }, + restoreSkippedOptions: [] + }) +} + +function exitBeforeProof(): Promise { + return host.handleAdapterEvent({ + type: 'ended', + ...currentChild(), + reason: EXIT_REASON, + cause: 'unexpected-exit', + startupUnproven: true + }) +} + +function journalStatuses(): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) +} + +function submission(clientMessageId: string) { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-send-after-failed-start-')) + resetHostTestOperationIds() + generation = 0 + acquire = vi.fn(async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + // A re-acquire resumes the thread the first child minted, as a real adapter does. + origin: generation === 0 ? ('created' as const) : ('resumed' as const), + mintedAtFence: fence, + observedAt: NOW + }, + acquisitionGeneration: `generation-${++generation}`, + providerChildPhase: 'starting' as const + })) + dispatch = vi.fn(async () => ({ state: 'admitted' as const })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + releaseAcquisition: vi.fn(async () => true), + closeSession: vi.fn(async () => true), + dispatch, + cancelTurn: vi.fn(async () => ({ cancelled: true })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined) + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${generation + 1}`, + now: () => NOW + }) + await expect(host.attach(CALLER, hostTestAttachParams(null))).resolves.toMatchObject({ + ok: true + }) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +describe('a send into a published session whose child ended before startup', () => { + beforeEach(async () => { + await exitBeforeProof() + // The failed start released the lease and no resume ran on its own. + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + expect(acquire).toHaveBeenCalledOnce() + }) + + it('restarts the child and admits the message against it before it has proven its start', async () => { + const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + + await send('hello again', releasedFence) + + // The client was current as of the lost owner, so the send is rebased onto the fence the + // resume published and admitted once, with no stale round trip. + expect(acquire).toHaveBeenCalledTimes(2) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + expect(dispatch).toHaveBeenCalledOnce() + const current = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + expect(current).toBeGreaterThan(releasedFence) + + // Once proven, the next send meets a live owner and restarts nothing. + await proveStarted() + await send('and again', current) + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).toHaveBeenCalledTimes(2) + }) + + it('retires the held message with the cause when the restarted child exits before proving its start', async () => { + const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + const rowsBefore = journalStatuses().length + + const held = await send('still not signed in', releasedFence) + await exitBeforeProof() + + // The child never proved its start, so it accepted nothing: the exit rejects the message this + // host admitted, so nothing pins the session and Retry stays offered, and one row names the cause. + expect(submission(held)).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining(EXIT_REASON), + recovered: true + }) + expect( + host.journalSnapshot(SESSION).submissions.filter((e) => e.dispatchState === 'pending') + ).toEqual([]) + expect(journalStatuses().slice(rowsBefore)).toEqual([ + expect.stringMatching(/stopped before it finished starting: .*not signed in/) + ]) + // The failed restart moved the fence twice: the acquisition, and the exit that released it. + expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(releasedFence + 2) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + // One spawn per user action: nothing restarted it a second time. + expect(acquire).toHaveBeenCalledTimes(2) + + // Retry is a fresh action: it restarts once and is admitted against the new child. + await send('signed in now') + expect(acquire).toHaveBeenCalledTimes(3) + expect(dispatch).toHaveBeenCalledTimes(2) + expect(journalStatuses().slice(rowsBefore)).toHaveLength(1) + }) +}) + +describe('a send while the child of the first start is still proving itself', () => { + it('is admitted against the starting child, and nothing restarts it', async () => { + await send('hello') + + expect(dispatch).toHaveBeenCalledOnce() + expect(acquire).toHaveBeenCalledOnce() + + await proveStarted() + + expect(acquire).toHaveBeenCalledOnce() + expect(journalStatuses()).toEqual([]) + }) + + it('is retired with the cause when that child exits first, and restarts nothing', async () => { + const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + const held = await send('hello') + + await exitBeforeProof() + + expect(submission(held)).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining(EXIT_REASON), + recovered: true + }) + expect(acquire).toHaveBeenCalledOnce() + expect(journalStatuses()).toEqual([ + expect.stringMatching(/stopped before it finished starting: .*not signed in/) + ]) + expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(fence + 1) + }) + + it('leaves a send against a proven child alone', async () => { + await proveStarted() + + await send('hello') + + expect(acquire).toHaveBeenCalledOnce() + expect(dispatch).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts index 278619c2c617..5780359a8aec 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts @@ -382,7 +382,10 @@ describe('settled attach retry', () => { it('records proven acquisition cleanup as durable death evidence', async () => { acquire.mockRejectedValueOnce(new Error('resume rejected')) - await expect(host.attach(CALLER, hostTestAttachParams(null))).rejects.toThrow('resume rejected') + await expect(host.attach(CALLER, hostTestAttachParams(null))).resolves.toMatchObject({ + ok: false, + refusal: { message: 'resume rejected', ownerVerdict: 'exited' } + }) expect(releaseAcquisition).toHaveBeenCalledTimes(1) expect(store.getRecord(SESSION)?.lease).toMatchObject({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-starting-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-starting-release.test.ts new file mode 100644 index 000000000000..df6e6cec4bf1 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-starting-release.test.ts @@ -0,0 +1,222 @@ +// A Claude chat is published before its CLI answers initialize, and a message sent in that window +// is held until it does. Switching away from the chat starts the release clock; the clock must +// treat that held message as work still owed, exactly as it treats a running turn, or it evicts +// the session and refuses a message the user already sent. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { ClaudeStructuredSessionAdapter } from '../../claude/claude-structured-session-adapter' +import { + fakeClaude, + PROVIDER_SESSION_ID +} from '../../claude/claude-structured-session-test-support' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +const SURFACE = 'desktop-chat:1' +const GRACE_MS = 5 + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let adapter: ClaudeStructuredSessionAdapter +let claude: ReturnType +let landInit: () => void +let lifecycle: Promise[] + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-starting-release-')) + resetHostTestOperationIds() + claude = fakeClaude() + lifecycle = [] + const initLanded = new Promise((resolve) => { + landInit = resolve + }) + adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: root, + claudeConfigDir: join(root, 'claude-home'), + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + resumesTranscript: false, + continuesChain: false + }), + onEvent: (event) => { + const mapped = structuredClaudeLifecycleEvent(event) + if (mapped) { + lifecycle.push(host.handleAdapterEvent(mapped)) + } + }, + // As the runtime wires it: a held prompt's outcome reaches the journal out of band. + onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), + // Initialize answers only when the test says so. + openConnection: async (launch, handlers) => { + const connection = await claude.openConnection(launch, handlers) + const answer = connection.initializationResult + connection.initializationResult = async () => { + await initLanded + return answer() + } + return connection + }, + readProcessStartTime: async () => 1_700_000_000_000, + now: () => NOW + }) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + adapter: Object.assign(adapter, { supportsCreate: () => true }), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + releaseGraceMs: GRACE_MS, + now: () => NOW + }) +}) + +afterEach(async () => { + vi.useRealTimers() + landInit() + await adapter.closeAll() + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +async function attachStarting(): Promise { + const created = await host.attach( + CALLER, + hostTestAttachParams(null, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION_ID, leafUuid: null } + }) + ) + expect(created).toMatchObject({ ok: true }) + await host.hold(SESSION, SURFACE) +} + +async function send(text: string, dispatchState = 'pending'): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState } } }) + return sent.ok ? sent.value.clientMessageId : '' +} + +function dispatchState(clientMessageId: string): string | undefined { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState +} + +/** Long enough for several grace windows to elapse, so "not evicted" means the clock declined. */ +function waitOutSeveralGraceWindows(): Promise { + return new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) +} + +describe('a chat left while its Claude CLI is still starting', () => { + it('keeps the session for a message it is holding, and delivers it once startup lands', async () => { + await attachStarting() + const held = await send('sent while starting') + + host.release(SESSION, SURFACE) + await waitOutSeveralGraceWindows() + + expect(host.hasSession(SESSION)).toBe(true) + expect(claude.connections[0].closeCount).toBe(0) + expect(dispatchState(held)).toBe('pending') + + landInit() + await adapter.drainStartup(SESSION) + + expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]) + await vi.waitFor(() => expect(dispatchState(held)).toBe('accepted')) + }) + + it('gives the message it wrote at startup a full grace to open its turn', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + await attachStarting() + const held = await send('sent while starting') + const connection = claude.connections[0] + // Claude echoes a prompt only when it starts that turn, which a loaded machine delays. + connection.send = async (message) => { + connection.sent.push(message) + } + host.release(SESSION, SURFACE) + await vi.advanceTimersByTimeAsync(GRACE_MS * 3 - 1) + expect(host.hasSession(SESSION)).toBe(true) + + // Startup lands just before the clock's next tick. + landInit() + await adapter.drainStartup(SESSION) + await Promise.all(lifecycle) + expect(connection.sent).toEqual([expect.objectContaining({ type: 'user' })]) + await vi.advanceTimersByTimeAsync(GRACE_MS - 1) + + expect(host.hasSession(SESSION)).toBe(true) + expect(connection.closeCount).toBe(0) + connection.handlers.onMessage?.(connection.sent[0]) + await host.flushStreamedEvents(SESSION) + await vi.advanceTimersByTimeAsync(GRACE_MS * 3) + expect(host.hasSession(SESSION)).toBe(true) + expect(dispatchState(held)).toBe('accepted') + }) + + it('is released after the grace once its turn has finished', async () => { + await attachStarting() + landInit() + await adapter.drainStartup(SESSION) + await send('answered', 'accepted') + claude.connections[0].handlers.onMessage?.({ + type: 'result', + subtype: 'success', + uuid: 'result-1', + session_id: PROVIDER_SESSION_ID, + is_error: false, + result: 'done' + }) + await host.flushStreamedEvents(SESSION) + expect(host.journalSnapshot(SESSION).submissions).toHaveLength(1) + + host.release(SESSION, SURFACE) + + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) + expect(claude.connections[0].closeCount).toBe(1) + }) + + it('is released after the grace when it owes nothing', async () => { + await attachStarting() + + host.release(SESSION, SURFACE) + + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) + expect(claude.connections[0].closeCount).toBe(1) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts new file mode 100644 index 000000000000..0fd544ec9eea --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../../shared/agent-session-record.test-fixture' +import { providerStartupFailureOutcome } from './structured-agent-session-dead-generation-settlement' +import { + settleUnexpectedStructuredAgentSessionExit, + type StructuredAgentSessionUnexpectedExitContext, + type StructuredAgentSessionUnexpectedExitSession +} from './structured-agent-session-unexpected-exit' + +const SESSION = 'session-1' +const GENERATION = 'generation-1' +const REASON = 'Claude Code is not signed in. Sign in with the Claude CLI' + +function startedSession(): StructuredAgentSessionUnexpectedExitSession & { + journal: { appendLifecycleBatch: ReturnType } +} { + return { + hasProviderChild: true, + fence: 7, + acquisitionGeneration: GENERATION, + journal: { + // Nothing ran: the start failed before any response or acknowledged prompt. + snapshot: () => ({ items: [] }), + appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), + markPendingSubmissionsUnknown: vi.fn(async () => []), + rejectPendingSubmissions: vi.fn(async () => []) + } + } +} + +function contextFor(session: StructuredAgentSessionUnexpectedExitSession) { + let record: AgentSessionRecord = agentSessionRecordFixture( + agentSessionLeaseFixture({ + sessionId: SESSION, + runtimeKind: 'native', + runtimeFence: 7, + handoffStage: null, + ownerProcess: { hostId: 'local', pid: 4242, processStartTimeMs: 1, spawnToken: 'spawn-1' }, + reservedSpawnToken: 'spawn-1', + claimStatus: 'live', + unreconciled: false + }) + ) + const context: StructuredAgentSessionUnexpectedExitContext = { + store: { + getRecord: () => record, + transitionHandoff: async ( + _sessionId: string, + transition: (current: AgentSessionRecord) => AgentSessionRecord + ) => (record = transition(record)) + }, + sessions: new Map([[SESSION, session]]), + flushLifecycle: async () => ({ ok: true }), + publishFence: vi.fn(), + hasResumeCapableHolder: () => true, + serialize: async (_sessionId: string, task: () => Promise) => task(), + now: () => 1 + } + return context +} + +const ended = { + type: 'ended' as const, + sessionId: SESSION, + reason: REASON, + cause: 'unexpected-exit' as const, + fence: 7, + acquisitionGeneration: GENERATION +} + +describe('a provider that ends before it finished starting', () => { + it('tells the user why, even with no response in progress, and does not auto-resume', async () => { + const session = startedSession() + + const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), { + ...ended, + startupUnproven: true + }) + + expect(ticket).toBeNull() + expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect.objectContaining({ + mutations: [ + expect.objectContaining({ + body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + }) + ] + }) + ) + expect(providerStartupFailureOutcome(REASON)).toContain('not signed in') + }) + + it('keeps an ordinary idle exit silent and resumable', async () => { + const session = startedSession() + + const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) + + expect(ticket).not.toBeNull() + expect(session.journal.appendLifecycleBatch).not.toHaveBeenCalled() + }) + + it("reads a start that failed off the host's own phase when the provider omits the flag", async () => { + const session = { ...startedSession(), providerChildPhase: 'starting' as const } + + const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) + + expect(ticket).toBeNull() + expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect.objectContaining({ + mutations: [ + expect.objectContaining({ + body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + }) + ] + }) + ) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts index 29e450c187d3..e4fa6ebed1bf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts @@ -3,6 +3,7 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store export function indexedStatusFeedSession(session: { journal: AgentSessionJournal hasProviderChild?: boolean + providerChildPhase?: 'starting' | 'ready' fence?: number }) { return { @@ -11,6 +12,7 @@ export function indexedStatusFeedSession(session: { ...(session.hasProviderChild !== undefined ? { hasProviderChild: session.hasProviderChild } : {}), + ...(session.providerChildPhase ? { providerChildPhase: session.providerChildPhase } : {}), params: { location: { executionHostId: 'local' as const, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index e5d3d16e5dd8..65c7f54fb305 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -60,10 +60,7 @@ async function openJournal(sessionId = SESSION, now?: () => number) { } function feedFor( - sessions: Map< - string, - { journal: Awaited>; hasProviderChild?: boolean; fence?: number } - >, + sessions: Map[0]>, record: Partial | null = null, onStatusChanged?: StructuredAgentSessionStatusFeedDeps['onStatusChanged'], readBackgroundTasks?: StructuredAgentSessionStatusFeedDeps['readBackgroundTasks'], @@ -94,6 +91,24 @@ function feedFor( } describe('StructuredAgentSessionStatusFeed', () => { + it('projects whether the owned child has proven its start, and nothing once it is not owned', async () => { + const journal = await openJournal() + const session = { journal, hasProviderChild: true, providerChildPhase: 'starting' as const } + const sessions = new Map[0]>([[SESSION, session]]) + const { feed, events, dispose } = feedFor(sessions) + expect(events.at(-1)).toMatchObject({ + type: 'snapshot', + sessions: [{ hostExecutionOwned: true, hostExecutionPhase: 'starting' }] + }) + sessions.set(SESSION, { ...session, providerChildPhase: 'ready' }) + feed.publish(SESSION, journal) + expect(events.at(-1)).toMatchObject({ session: { hostExecutionPhase: 'ready' } }) + sessions.set(SESSION, { ...session, hasProviderChild: false }) + feed.publish(SESSION, journal) + expect(events.at(-1)).not.toMatchObject({ session: { hostExecutionPhase: expect.any(String) } }) + dispose() + }) + it('publishes provider ownership transitions without changing journal time', async () => { const journal = await openJournal() const sessions = new Map([[SESSION, { journal, hasProviderChild: true }]]) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index 6ec254f1032e..4fefea8639ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -22,6 +22,7 @@ import { } from '../../../shared/agent-session-wire' import { projectStructuredAgentSessionStatusSummary } from '../../../shared/structured-agent-session-projection' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionProviderChildPhase } from './structured-agent-session-adapter' import { structuredAgentSessionProviderSessionMetadata } from './structured-agent-session-history-result' import { StructuredAgentSessionStatusOwnership, @@ -39,6 +40,7 @@ type StatusFeedSession = { journal: AgentSessionJournal params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } hasProviderChild?: boolean + providerChildPhase?: StructuredAgentSessionProviderChildPhase fence?: number } @@ -63,6 +65,7 @@ function summariesEqual(a: AgentSessionStatusSummary, b: AgentSessionStatusSumma a.agent === b.agent && a.status === b.status && a.hostExecutionOwned === b.hostExecutionOwned && + a.hostExecutionPhase === b.hostExecutionPhase && a.rewindBlockedReason === b.rewindBlockedReason && // Settled activity changes ranking; streaming active turns must stay quiet. (a.status !== 'idle' || a.updatedAt === b.updatedAt) && @@ -254,7 +257,14 @@ export class StructuredAgentSessionStatusFeed { sessionId, workspaceId: session.params.location.workspaceId, agent: session.params.provider, - ...(session.hasProviderChild ? { hostExecutionOwned: true as const } : {}), + ...(session.hasProviderChild + ? { + hostExecutionOwned: true as const, + ...(session.providerChildPhase + ? { hostExecutionPhase: session.providerChildPhase } + : {}) + } + : {}), ...projection.summary, ...(record?.rewind?.phase === 'prepared' || record?.rewind?.phase === 'provider-succeeded' ? { rewindBlockedReason: 'outcome-unknown' as const } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index e26e65d1db45..2071b74ed459 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -431,6 +431,17 @@ describe('a session with a turn in flight', () => { await waitForEviction() }) + + // Codex settles an admitted send only on its echo, which may never come; eviction retires it. + it('is evicted with an admitted send outstanding once no turn runs', async () => { + await attach() + await host.hold(SESSION, SURFACE) + await sendPending('admitted, never echoed') + + host.release(SESSION, SURFACE) + + await waitForEviction() + }) }) describe('startup', () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index 4c9cc2213d5c..28d7846f10d2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -20,8 +20,10 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store import { latestJournalDispatchObservation } from '../agent-session-journal/journal-dispatch-observation' import type { AgentSessionDispatchOutcome, - StructuredAgentSessionAdapter + StructuredAgentSessionAdapter, + StructuredAgentSessionProviderChildPhase } from './structured-agent-session-adapter' +import { providerStartupFailureRejection } from './structured-agent-session-dead-generation-settlement' import { validatePendingPrompt } from './structured-agent-session-prompt-state' import { withTimeout } from '../../../shared/promise-timeout-fallback' import { @@ -46,6 +48,8 @@ export type AgentSessionTurnContext = { hasPendingStreamedEvents?: () => boolean /** Re-derives authorization after submission persistence, immediately before provider dispatch. */ beforeDispatch?: () => void + /** What the host holds about the child this dispatch is for, read at the moment it is needed. */ + providerChildPhase?: () => StructuredAgentSessionProviderChildPhase | undefined now: () => number } @@ -57,8 +61,10 @@ function invalid(message: string): { ok: false; refusal: AgentSessionWireRefusal return { ok: false, refusal: { code: 'agent_session_operation_invalid', message } } } -/** A thrown adapter error is indistinguishable from a lost reply, so it settles - * as `unknown` rather than as a rejection. */ +/** A thrown adapter error is indistinguishable from a lost reply, so it settles as `unknown` + * rather than as a rejection — unless the child had not proven its start. Such a child has + * accepted nothing (input is written only after it initializes), so a dispatch it could not + * take is provably unwritten and is rejected with the cause the adapter gave. */ async function dispatchSafely( ctx: AgentSessionTurnContext, clientMessageId: string, @@ -95,6 +101,9 @@ async function dispatchSafely( if (error instanceof AgentSessionPreDispatchError) { throw error } + if (ctx.providerChildPhase?.() === 'starting') { + return { state: 'rejected', reason: providerStartupFailureRejection(error) } + } return { state: 'unknown', reason: error instanceof Error ? error.message : String(error) } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 8dc2112b18ad..e864a1bc22b5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -311,7 +311,8 @@ describe('provider-exit recovery tickets', () => { journal: { snapshot: () => ({ items }), appendLifecycleBatch, - markPendingSubmissionsUnknown: vi.fn(async () => []) + markPendingSubmissionsUnknown: vi.fn(async () => []), + rejectPendingSubmissions: vi.fn(async () => []) } } @@ -368,6 +369,7 @@ describe('provider-exit recovery tickets', () => { snapshot: () => ({ items: [] }), appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), markPendingSubmissionsUnknown, + rejectPendingSubmissions: vi.fn(async () => []), submissions: () => [{ clientMessageId: 'client-1', dispatchState: 'pending' }] } } @@ -413,6 +415,7 @@ describe('provider-exit recovery tickets', () => { acquisitionGeneration: GENERATION, journal: { markPendingSubmissionsUnknown: vi.fn(async () => []), + rejectPendingSubmissions: vi.fn(async () => []), snapshot: () => ({ items: [lifecycleItem('turn-failing', 1, { state: 'running', startedAt: 1 })] }), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts index 11d004089ca6..06740bfab25c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts @@ -1,4 +1,7 @@ -import type { StructuredAgentSessionLifecycleEvent } from './structured-agent-session-adapter' +import type { + StructuredAgentSessionEndedEvent, + StructuredAgentSessionProviderChildPhase +} from './structured-agent-session-adapter' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit, @@ -14,7 +17,7 @@ import { } from './structured-agent-session-dead-generation-settlement' import type { StructuredAgentSessionTurnVerdict } from './structured-agent-session-stale-turn-verdict' -type UnexpectedExitLifecycleEvent = StructuredAgentSessionLifecycleEvent & { +type UnexpectedExitLifecycleEvent = StructuredAgentSessionEndedEvent & { cause: 'unexpected-exit' } @@ -30,6 +33,7 @@ export type StructuredAgentSessionUnexpectedExitSession = { hasProviderChild: boolean fence: number acquisitionGeneration: string | null + providerChildPhase?: StructuredAgentSessionProviderChildPhase } export type StructuredAgentSessionUnexpectedExitContext< @@ -50,7 +54,7 @@ export async function settleUnexpectedStructuredAgentSessionExit< TSession extends StructuredAgentSessionUnexpectedExitSession >( context: StructuredAgentSessionUnexpectedExitContext, - event: StructuredAgentSessionLifecycleEvent + event: StructuredAgentSessionEndedEvent ): Promise { if (event.cause !== 'unexpected-exit') { return null @@ -74,6 +78,10 @@ export async function settleUnexpectedStructuredAgentSessionExit< context.publishStatus?.(unexpectedEvent.sessionId) return null } + // The host's own phase decides, so a provider that omits the flag still gets a start that + // failed told as one: the row says so, and nothing resumes into the same failure. + const exitedDuringStartup = + unexpectedEvent.startupUnproven === true || session.providerChildPhase === 'starting' let settlementFailed = false const stableSettlementId = providerExitSettlementId(unexpectedEvent) @@ -96,11 +104,15 @@ export async function settleUnexpectedStructuredAgentSessionExit< session, stableSettlementId, verdict: { state: 'interrupted', completedAt: observedAt }, - showUnexpectedExitOutcome: unfinishedStructuredAgentSessionWorkWasInterrupted( - unfinishedWork, - session.journal, - observedAt - ) + exitedDuringStartup, + // A failed start always says why: no response was running to carry the reason. + showUnexpectedExitOutcome: + exitedDuringStartup || + unfinishedStructuredAgentSessionWorkWasInterrupted( + unfinishedWork, + session.journal, + observedAt + ) })) } finally { // Provider exit was positively observed, so release the owner even when @@ -138,7 +150,8 @@ export async function settleUnexpectedStructuredAgentSessionExit< if (settlementFailed || !released) { return null } - if (!context.hasResumeCapableHolder(unexpectedEvent.sessionId)) { + // Resuming a start that failed would respawn into the same failure; the next send retries. + if (exitedDuringStartup || !context.hasResumeCapableHolder(unexpectedEvent.sessionId)) { return null } return { @@ -183,6 +196,7 @@ async function retryUnexpectedExitSettlement(input: { session: Pick stableSettlementId: string verdict: StructuredAgentSessionTurnVerdict + exitedDuringStartup: boolean showUnexpectedExitOutcome?: boolean }): Promise { return settleStructuredAgentSessionDeadGeneration({ @@ -194,6 +208,7 @@ async function retryUnexpectedExitSettlement(input: { pendingSubmissionReason: 'provider_exited_before_acknowledgement', showUnexpectedExitOutcome: input.showUnexpectedExitOutcome, unexpectedExitReason: input.event.reason, + exitedDuringStartup: input.exitedDuringStartup, onError: input.context.onBarrierError }) } diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts index 68939c0889df..a261c6dd1517 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts @@ -163,6 +163,30 @@ describe('host conversation commands', () => { }) }) + // What the child reports can be a value it fell back to, such as a model whose restore write it + // never answered; the replacement's start replays the choice, as the source's next start would. + it('starts the replacement from the options the user chose, not the values the child reports', async () => { + await store.replaceSessionOptions({ + sessionId: HOST_TEST_SESSION, + fence: store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence, + options: { model: 'test-model', effort: 'low' }, + now: HOST_TEST_NOW + }) + adapter.readOptions = async () => ({ + models: [], + current: { model: 'fallback-model', effort: 'high' } + }) + const attach = vi.spyOn(host, 'attach') + expect(await host.conversationCommand(caller, commandParams('clear'))).toMatchObject({ + ok: true + }) + expect(attach.mock.calls[0]?.[1].options).toEqual({ model: 'test-model', effort: 'low' }) + expect(store.getRecord(HOST_TEST_SESSION)?.options).toEqual({ + model: 'test-model', + effort: 'low' + }) + }) + it('clears with a fresh record and effective options, retaining old history and idempotent mapping', async () => { const before = store.getRecord(HOST_TEST_SESSION)! const params = commandParams('clear') diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts index 44131886f5b2..022343d94070 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts @@ -52,7 +52,7 @@ export function runStructuredConversationCommand( adapter: context.deps.adapter, callerKey: caller.callerKey, envelope, - journal: context.sessions.get(sessionId)?.journal, + journal: () => context.sessions.get(sessionId)?.journal, publish: (journal) => context.publish(sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, now: context.now, @@ -112,33 +112,6 @@ export function runStructuredConversationCommand( state: 'unknown' as const, ...(replacementSessionId ? { replacementSessionId } : {}) } - let effectiveOptions = record.options - if (command === 'clear' && !prior) { - try { - const options = await ctx.adapter.readOptions?.({ sessionId, fence: ctx.fence }) - effectiveOptions = { - ...record.options, - ...(options - ? { - model: options.current.model, - ...(options.current.effort ? { effort: options.current.effort } : {}) - } - : {}) - } - } catch { - return { - ok: false, - refusal: { - code: 'agent_session_operation_invalid', - message: - 'Could not read the current session configuration. Try again when the provider is connected.' - } - } - } - } - if (effectiveOptions && command === 'clear') { - await ctx.persistOptions(effectiveOptions) - } await store.setConversationCommand(sessionId, ctx.fence, prepared) let error: string | undefined if (command === 'clear' && replacementSessionId) { @@ -160,7 +133,8 @@ export function runStructuredConversationCommand( agent: record.provider, runtimeKind: 'native', launchArgs: record.launchArgs, - options: effectiveOptions + // The options the user chose, which any restart of this chat would replay too. + options: record.options } attach.envelope.payloadFingerprint = computeAgentSessionPayloadFingerprint({ method: 'agentSession.attach', diff --git a/src/main/native-chat/agent-session-wire/structured-tui-transcript-catchup.test.ts b/src/main/native-chat/agent-session-wire/structured-tui-transcript-catchup.test.ts index 6e4cf64b5bfc..9866b811c613 100644 --- a/src/main/native-chat/agent-session-wire/structured-tui-transcript-catchup.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-tui-transcript-catchup.test.ts @@ -104,6 +104,7 @@ function createCatchup(input: Awaited>) store, session: () => ({ hasProviderChild: false, + providerChildPhase: 'ready', journal: input.journal, params: {} as never, fence: input.fence, diff --git a/src/main/observability/agent-session-instrumentation.ts b/src/main/observability/agent-session-instrumentation.ts index 85ecd4f0d452..dc57bd69d0e4 100644 --- a/src/main/observability/agent-session-instrumentation.ts +++ b/src/main/observability/agent-session-instrumentation.ts @@ -9,7 +9,6 @@ export type AgentSessionCreatePhase = | 'acquire_owner' | 'auth_settle' | 'spawn' - | 'init' | 'restore_options' | 'publish' diff --git a/src/main/observability/instrumentation.test.ts b/src/main/observability/instrumentation.test.ts index 62894c0ef9c1..9bdbfd91d026 100644 --- a/src/main/observability/instrumentation.test.ts +++ b/src/main/observability/instrumentation.test.ts @@ -258,7 +258,7 @@ describe('agentSession.create tracing', () => { it('emits one span with the closed phase vocabulary and no user content attributes', async () => { await withAgentSessionSpan(async (span) => { addAgentSessionCreatePhaseAttributes(span, { - totalDurationMs: 66, + totalDurationMs: 57, phases: [ { phase: 'reconcile_leases', startedAtMs: 0, durationMs: 1 }, { phase: 'resolve_recovery', startedAtMs: 1, durationMs: 2 }, @@ -268,9 +268,8 @@ describe('agentSession.create tracing', () => { { phase: 'acquire_owner', startedAtMs: 15, durationMs: 6 }, { phase: 'auth_settle', startedAtMs: 21, durationMs: 7 }, { phase: 'spawn', startedAtMs: 28, durationMs: 8 }, - { phase: 'init', startedAtMs: 36, durationMs: 9 }, - { phase: 'restore_options', startedAtMs: 45, durationMs: 10 }, - { phase: 'publish', startedAtMs: 55, durationMs: 11 } + { phase: 'restore_options', startedAtMs: 36, durationMs: 10 }, + { phase: 'publish', startedAtMs: 46, durationMs: 11 } ] }) }) diff --git a/src/main/runtime/agent-session-operation-admission.ts b/src/main/runtime/agent-session-operation-admission.ts index a4d90216651d..49dbab1c90dc 100644 --- a/src/main/runtime/agent-session-operation-admission.ts +++ b/src/main/runtime/agent-session-operation-admission.ts @@ -5,6 +5,7 @@ import { agentSessionOperationKey, claimAgentSessionOperation, evaluateAgentSessionOperation, + findAgentSessionGlobalOperationRow, pruneAgentSessionOperationRows, settleAgentSessionOperation, type AgentSessionOperationClaim, @@ -42,60 +43,105 @@ export type AgentSessionMutationOperationDecision = { record: AgentSessionRecord } | null -/** Prune, evaluate, and (on admit) place the row. The caller runs this inside one - * transaction, so two concurrent copies of an operation id cannot both admit. */ -export function admitAgentSessionOperationRow( +type EvaluatedOperationRows = { rows: OperationRows; decision: AgentSessionOperationDecision } + +/** Prune and evaluate, placing nothing: the ledger's answer as it stands. */ +export function evaluateAgentSessionOperationRow( rows: OperationRows, args: AgentSessionOperationAdmission -): { rows: OperationRows; decision: AgentSessionOperationDecision } { +): EvaluatedOperationRows { const pruned = pruneAgentSessionOperationRows(rows, args.now) - const decision = evaluateAgentSessionOperation({ rows: pruned, ...args }) - if (decision.decision === 'admit') { - pruned.set(agentSessionOperationKey(args.callerKey, args.operationId), decision.row) - } - return { rows: pruned, decision } + return { rows: pruned, decision: evaluateAgentSessionOperation({ rows: pruned, ...args }) } } /** Send ids name one provider delivery even when the authenticated caller changes. */ -export function admitAgentSessionGlobalOperationRow( +export function evaluateAgentSessionGlobalOperationRow( rows: OperationRows, args: AgentSessionOperationAdmission -): { rows: OperationRows; decision: AgentSessionOperationDecision } { - let existing: AgentSessionOperationRow | undefined - for (const row of rows.values()) { - if (row.expiresAt > args.now && row.operationId === args.operationId) { - existing = row - break - } - } +): EvaluatedOperationRows { + const existing = findAgentSessionGlobalOperationRow(rows, args.operationId, args.now) if (!existing) { - return admitAgentSessionOperationRow(rows, args) + return evaluateAgentSessionOperationRow(rows, args) } - const pruned = pruneAgentSessionOperationRows(rows, args.now) const syntheticRows = new Map([ [agentSessionOperationKey(args.callerKey, args.operationId), existing] ]) return { - rows: pruned, + rows: pruneAgentSessionOperationRows(rows, args.now), decision: evaluateAgentSessionOperation({ rows: syntheticRows, ...args }) } } -/** Admit the ledger row and its lease/fence preconditions in one durable transaction. */ -export function admitAgentSessionMutationOperation( - state: AgentSessionStoreState, +/** Places the row an evaluation admitted. The caller runs this inside one transaction, so two + * concurrent copies of an operation id cannot both admit. */ +function placeAdmittedAgentSessionOperationRow( + evaluated: EvaluatedOperationRows, + args: AgentSessionOperationAdmission +): EvaluatedOperationRows { + if (evaluated.decision.decision === 'admit') { + evaluated.rows.set( + agentSessionOperationKey(args.callerKey, args.operationId), + evaluated.decision.row + ) + } + return evaluated +} + +export function admitAgentSessionOperationRow( + rows: OperationRows, + args: AgentSessionOperationAdmission +): EvaluatedOperationRows { + return placeAdmittedAgentSessionOperationRow(evaluateAgentSessionOperationRow(rows, args), args) +} + +export function admitAgentSessionGlobalOperationRow( + rows: OperationRows, + args: AgentSessionOperationAdmission +): EvaluatedOperationRows { + return placeAdmittedAgentSessionOperationRow( + evaluateAgentSessionGlobalOperationRow(rows, args), + args + ) +} + +/** The ledger's answer for a mutation, placing nothing and checking no lease: what a call must + * know before it decides whether to give the session an owner. Null when no record exists. */ +export function evaluateAgentSessionMutationOperation( + state: Pick, args: AgentSessionMutationOperationAdmission -): AgentSessionMutationOperationDecision { +): { decision: AgentSessionOperationDecision; record: AgentSessionRecord } | null { const record = state.records.get(args.envelope.sessionId) if (!record) { return null } - const operation = { + const operation = mutationOperation(args) + const evaluated = args.operationIdScope + ? evaluateAgentSessionGlobalOperationRow(state.operations, operation) + : evaluateAgentSessionOperationRow(state.operations, operation) + return { decision: evaluated.decision, record } +} + +function mutationOperation( + args: AgentSessionMutationOperationAdmission +): AgentSessionOperationAdmission { + return { callerKey: args.callerKey, operationId: args.envelope.clientOperationId, fingerprint: args.hostFingerprint, now: args.now } +} + +/** Admit the ledger row and its lease/fence preconditions in one durable transaction. */ +export function admitAgentSessionMutationOperation( + state: AgentSessionStoreState, + args: AgentSessionMutationOperationAdmission +): AgentSessionMutationOperationDecision { + const record = state.records.get(args.envelope.sessionId) + if (!record) { + return null + } + const operation = mutationOperation(args) const ledger = args.operationIdScope ? admitAgentSessionGlobalOperationRow(state.operations, operation) : admitAgentSessionOperationRow(state.operations, operation) diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index 807547af3c30..22d93d6c886b 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -3,15 +3,17 @@ import { commitConversationCommandRecord } from './agent-session-conversation-co import { setAgentSessionRecordConversationName } from './agent-session-record-conversation-name' /** Durable single-writer session records and their operation ledger. */ -import type { - AgentSessionOperationClaim, - AgentSessionOperationDecision, - AgentSessionOperationOutcome, - AgentSessionOperationRow +import { + agentSessionOperationKey, + type AgentSessionOperationClaim, + type AgentSessionOperationDecision, + type AgentSessionOperationOutcome, + type AgentSessionOperationRow } from '../../shared/agent-session-operation-ledger' import { admitAgentSessionGlobalOperationInto, admitAgentSessionMutationOperation, + evaluateAgentSessionMutationOperation, admitAgentSessionOperationInto, claimAgentSessionOperationInto, settleAgentSessionOperationInto, @@ -163,6 +165,9 @@ export class AgentSessionRecordStore { listOperationRows = (): AgentSessionOperationRow[] => [...this.state.operations.values()] + getOperationRow = (callerKey: string, operationId: string): AgentSessionOperationRow | null => + this.state.operations.get(agentSessionOperationKey(callerKey, operationId)) ?? null + isClaimKeyVerifiable = (keyId: string, now: number): boolean => isAgentSessionClaimKeyVerifiable(this.state, keyId, now) @@ -287,6 +292,10 @@ export class AgentSessionRecordStore { admitMutationOperation = (args: AgentSessionMutationOperationAdmission) => this.transact(() => admitAgentSessionMutationOperation(this.state, args)) + /** The ledger's answer alone, placing nothing; `admitMutationOperation` is the transaction. */ + evaluateMutationOperation = (args: AgentSessionMutationOperationAdmission) => + evaluateAgentSessionMutationOperation(this.state, args) + /** Durable compare-and-swap for the right to run an admitted operation's effect: two replays both * read `pending`, and only a conditional swap tells the one that may run from the one that must * replay. */ diff --git a/src/main/runtime/agent-session-reservation-admission.test.ts b/src/main/runtime/agent-session-reservation-admission.test.ts index 80bcb77cfa5b..30d54a108c9c 100644 --- a/src/main/runtime/agent-session-reservation-admission.test.ts +++ b/src/main/runtime/agent-session-reservation-admission.test.ts @@ -197,3 +197,48 @@ describe('adopted conversation ownership', () => { ).toThrow('agent_session_conflict') }) }) + +describe('re-create over a failed create', () => { + const EXITED = agentSessionLeaseFixture({ + sessionId: 'session-adopting', + runtimeKind: 'native', + runtimeFence: 2, + provenHandleLinkId: null, + ownerProcess: null, + reservedSpawnToken: null, + claimStatus: 'released' + }) + function failedCreate(overrides: Partial = {}): AgentSessionRecord { + return { + ...agentSessionRecordFixture(EXITED), + location: LOCATION, + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + providerHandleChain: [], + ...overrides + } + } + + it('reserves a record that never bound a conversation and whose attempt is proven gone', () => { + const { record, disposition } = applyAgentSessionReservation( + storeState([failedCreate()]), + reserveRequest(), + LEASE_TTL_MS + ) + + expect(disposition).toBe('reserved') + expect(record.lease).toMatchObject({ claimStatus: 'reserved', runtimeFence: 3 }) + }) + + it('refuses when the record bound a conversation, or its attempt may still run', () => { + const bound = failedCreate({ providerHandleChain: [adoptedLink()] }) + const unproven = failedCreate({ + lease: { ...EXITED, claimStatus: 'reserved', handoffStage: 'manual-recovery' } + }) + + for (const record of [bound, unproven]) { + expect(() => + applyAgentSessionReservation(storeState([record]), reserveRequest(), LEASE_TTL_MS) + ).toThrow('agent_session_conflict') + } + }) +}) diff --git a/src/main/runtime/agent-session-reservation-admission.ts b/src/main/runtime/agent-session-reservation-admission.ts index 662ea481921d..436267e24eac 100644 --- a/src/main/runtime/agent-session-reservation-admission.ts +++ b/src/main/runtime/agent-session-reservation-admission.ts @@ -17,6 +17,7 @@ import { type AgentSessionOperationRow } from '../../shared/agent-session-operation-ledger' import { + agentSessionLeaseOwnerVerdict, evaluateAgentSessionAcquisition, type AgentSessionOwnerProbe } from '../../shared/agent-session-lease-adjudication' @@ -181,7 +182,13 @@ export function applyAgentSessionReservation( // Why: location, provider, and account are the session identity; changing one is a fork. throw new Error('agent_session_conflict') } - if (request.expectedFence === null) { + // A create may take over only a record that never bound a conversation and whose last + // attempt is proven gone: that is the same as creating it fresh, under a fresh provider id. + const recreatable = + existing.providerHandleChain.length === 0 && + !request.adoptedHandleLink && + agentSessionLeaseOwnerVerdict(existing.lease) === 'exited' + if (request.expectedFence === null && !recreatable) { throw new Error('agent_session_conflict') } const pinned = { @@ -191,7 +198,7 @@ export function applyAgentSessionReservation( } return reserveAgentSessionOwner({ record: pinned, - expectedFence: request.expectedFence, + expectedFence: request.expectedFence ?? existing.lease.runtimeFence, probe: request.probe, reservation }) diff --git a/src/main/runtime/claude-structured-failed-start-resume.test.ts b/src/main/runtime/claude-structured-failed-start-resume.test.ts new file mode 100644 index 000000000000..32362cbafdce --- /dev/null +++ b/src/main/runtime/claude-structured-failed-start-resume.test.ts @@ -0,0 +1,72 @@ +// A Claude chat whose first start died before initialize (not signed in, a crash) wrote no +// transcript, so its next start launches that same provider id fresh. It is still the same +// conversation: the new child continues the record's chain rather than creating a second root. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { claudeSessionIdForOrcaSession } from '../claude/claude-structured-launch-resolution' +import { waitForStructuredAgentSessionRecovery } from './structured-agent-session-runtime' +import { createScriptedClaudeRuntime } from './structured-claude-scripted-runtime-test-support' + +const SESSION = 'claude-failed-start' +const CALLER = { callerKey: 'client-1' } + +let claude = createScriptedClaudeRuntime([SESSION]) + +afterEach(async () => { + await claude.dispose() + claude = createScriptedClaudeRuntime([SESSION]) +}) + +describe('a Claude chat whose first start died before initialize', () => { + it('resumes on reopen, launching its id fresh and continuing the same chain', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + claude.child(SESSION).exit(new Error('claude stream-json exited (code 1): not signed in')) + await waitForStructuredAgentSessionRecovery() + const failed = host.deps.store.getRecord(SESSION) + expect(failed?.lease.claimStatus).toBe('released') + + // The user signs in and reopens the chat. + claude.behave(SESSION, {}) + const reopened = await host.attach( + CALLER, + claude.attachParams(SESSION, failed?.lease.runtimeFence ?? null) + ) + + expect(reopened, JSON.stringify(reopened)).toMatchObject({ ok: true }) + const providerSessionId = claudeSessionIdForOrcaSession(SESSION) + // No transcript exists to `--resume`, so the id is started fresh... + expect(claude.child(SESSION).launch.options).toMatchObject({ sessionId: providerSessionId }) + expect(claude.child(SESSION).launch.options.resume).toBeUndefined() + // ...and the record still holds one conversation under one root. + expect( + host.deps.store.getRecord(SESSION)?.providerHandleChain.map((link) => link.origin) + ).toEqual(['created', 'resumed']) + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + ) + }) +}) + +describe('a Claude chat whose CLI exits the moment it is spawned', () => { + // Before the spawn returns, the start time of a dead pid is unreadable; during that read, the + // child is found closed afterwards. Both must answer with what the CLI said. + it.each(['spawn', 'start-time-read'] as const)( + "refuses the create with the CLI's own diagnostic when it exits at %s", + async (at) => { + const diagnostic = 'claude stream-json exited (code 1): claude: not signed in' + claude.behave(SESSION, { exitsDuringSpawn: { diagnostic, at } }) + const host = await claude.install() + + const created = await host.attach(CALLER, claude.attachParams(SESSION, null)) + + expect(created).toMatchObject({ + ok: false, + refusal: { message: expect.stringContaining('not signed in'), ownerVerdict: 'exited' } + }) + } + ) +}) diff --git a/src/main/runtime/claude-structured-fake-connection-test-fixture.ts b/src/main/runtime/claude-structured-fake-connection-test-fixture.ts index 803d01f6e251..b64f6980689d 100644 --- a/src/main/runtime/claude-structured-fake-connection-test-fixture.ts +++ b/src/main/runtime/claude-structured-fake-connection-test-fixture.ts @@ -89,7 +89,7 @@ export function fakeClaude(providerSession: string) { exitVerdict: selfExit?.exitVerdict ?? { root: 'live', tree: 'unverifiable' }, close: async () => { connection.closed = true - return selfExit === null + return selfExit === null || selfExit.exitVerdict.tree === 'exited' } } connections.push(connection) diff --git a/src/main/runtime/claude-structured-resumed-start-failure.test.ts b/src/main/runtime/claude-structured-resumed-start-failure.test.ts new file mode 100644 index 000000000000..8fcef7e4a48b --- /dev/null +++ b/src/main/runtime/claude-structured-resumed-start-failure.test.ts @@ -0,0 +1,79 @@ +// A reopened Claude chat is published as soon as its child spawns, so a CLI that dies before it +// answers initialize fails a session the user is already looking at. That chat must say why, in +// the transcript, exactly as a failed first start does. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope' +import type { AgentSessionSubscribeEvent } from '../../shared/agent-session-wire' +import { hostTestMessage } from '../native-chat/agent-session-wire/structured-agent-session-host-test-data' +import { waitForStructuredAgentSessionRecovery } from './structured-agent-session-runtime' +import { createScriptedClaudeRuntime } from './structured-claude-scripted-runtime-test-support' + +const SESSION = 'claude-resumed-start' +const CALLER = { callerKey: 'client-1' } +const DIAGNOSTIC = 'claude stream-json exited (code 1): claude: not signed in' + +let claude = createScriptedClaudeRuntime([SESSION]) + +afterEach(async () => { + await claude.dispose() + claude = createScriptedClaudeRuntime([SESSION]) +}) + +/** Status rows the open chat was sent, whether in a batch or in the snapshot a fence change sends. */ +function statusTexts(events: AgentSessionSubscribeEvent[]): string[] { + return events.flatMap((event) => { + const items = + event.type === 'batch' ? event.batch.items : event.type === 'snapshot' ? event.page.items : [] + return items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) + }) +} + +describe('a reopened Claude chat whose CLI dies before initialize', () => { + it('publishes the startup failure, with the diagnostic, to the open chat', async () => { + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + await waitForStructuredAgentSessionRecovery() + await host.close(SESSION) + + // The user reopens it; this time the CLI never answers, then dies, and its tree is unprovable. + claude.behave(SESSION, { initHangs: true, closeUnproven: true }) + await host.hold(SESSION, 'surface-1') + const events: AgentSessionSubscribeEvent[] = [] + host.subscribe({ id: 'sub-1', sessionId: SESSION, emit: (event) => events.push(event) }) + const body = hostTestMessage('hello') + const fence = host.deps.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + const sent = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: `${Date.now()}-${'f'.repeat(32)}`, + expectedRuntimeFence: fence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + + claude.child(SESSION).exit(new Error(DIAGNOSTIC)) + await waitForStructuredAgentSessionRecovery() + + await vi.waitFor(() => + expect(statusTexts(events)).toContainEqual( + expect.stringMatching(/stopped before it finished starting: .*not signed in/) + ) + ) + // Never written, so it did not happen: refused, not left in doubt. + const submission = host + .journalSnapshot(SESSION) + .submissions.find( + (entry) => entry.clientMessageId === (sent.ok && sent.value.clientMessageId) + ) + expect(submission).toMatchObject({ dispatchState: 'rejected' }) + }) +}) diff --git a/src/main/runtime/claude-structured-send-held-for-startup.test.ts b/src/main/runtime/claude-structured-send-held-for-startup.test.ts new file mode 100644 index 000000000000..19e058110553 --- /dev/null +++ b/src/main/runtime/claude-structured-send-held-for-startup.test.ts @@ -0,0 +1,164 @@ +// A Claude chat is published the moment its child spawns, before the CLI has answered initialize. +// A send in that window — into a fresh start, or into the restart a send itself asked for after +// a start that failed — is admitted and held until the child proves its start. When the CLI dies +// first, the held message is rejected with the CLI's own diagnostic, the chat shows the cause +// once, and nothing is left as a delivery nobody can confirm. Against the production runtime, +// adapter, record store and host, with only the CLI process scripted. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope' +import { hostTestMessage } from '../native-chat/agent-session-wire/structured-agent-session-host-test-data' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { waitForStructuredAgentSessionRecovery } from './structured-agent-session-runtime' +import { createScriptedClaudeRuntime } from './structured-claude-scripted-runtime-test-support' + +const SESSION = 'claude-send-held' +const CALLER = { callerKey: 'client-1' } +const DIAGNOSTIC = 'claude stream-json exited (code 1): claude: not signed in (rig)' + +let claude = createScriptedClaudeRuntime([SESSION]) +let operations = 0 + +afterEach(async () => { + await claude.dispose() + claude = createScriptedClaudeRuntime([SESSION]) +}) + +async function send(host: StructuredAgentSessionHost, text: string): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: `${Date.now()}-${(++operations).toString(16).padStart(32, '0')}`, + expectedRuntimeFence: fence(host), + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + // Admitted and held for the start, never refused: the message shows as sent. + expect(sent, JSON.stringify(sent)).toMatchObject({ + ok: true, + replayed: false, + value: { submission: { dispatchState: 'pending' } } + }) + return sent.ok ? sent.value.clientMessageId : '' +} + +function fence(host: StructuredAgentSessionHost): number { + return host.deps.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 +} + +function statusRows(host: StructuredAgentSessionHost): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) +} + +function submission(host: StructuredAgentSessionHost, clientMessageId: string) { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId) +} + +/** The CLI keeps dying at startup: the latest child exits with the diagnostic once it exists. */ +async function failLatestStart(host: StructuredAgentSessionHost, count: number): Promise { + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(count)) + claude.child(SESSION).exit(new Error(DIAGNOSTIC)) + await waitForStructuredAgentSessionRecovery() + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + ) +} + +describe('a send into a Claude chat whose CLI keeps failing at startup', () => { + it('restarts once, rejects the held message with the diagnostic when that start dies too, then delivers once the CLI is healthy', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + await failLatestStart(host, 1) + expect(statusRows(host)).toEqual([expect.stringContaining('not signed in')]) + const releasedFence = fence(host) + + // The send asks for the child back and is held for its start; the CLI dies again first. + const held = await send(host, 'hello?') + expect(claude.children(SESSION)).toHaveLength(2) + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + await failLatestStart(host, 2) + + // Rejected with the cause, not left in doubt; one row for this attempt names it. + await vi.waitFor(() => + expect(submission(host, held)).toMatchObject({ + dispatchState: 'rejected', + // Worded for the user: the red line under the composer shows it as it stands. + reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + }) + ) + expect(statusRows(host)).toEqual([ + expect.stringContaining('not signed in'), + expect.stringMatching(/stopped before it finished starting: .*not signed in \(rig\)/) + ]) + // The restart moved the fence twice: its acquisition, and the exit that released it. + expect(fence(host)).toBe(releasedFence + 2) + expect(claude.children(SESSION)).toHaveLength(2) + expect(claude.child(SESSION).calls).not.toContain('send') + + // The user signs in and retries: one restart, proven, written to the CLI. + claude.behave(SESSION, {}) + await send(host, 'hello again') + expect(claude.children(SESSION)).toHaveLength(3) + expect(fence(host)).toBe(releasedFence + 3) + await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + ) + expect(statusRows(host)).toHaveLength(2) + }) +}) + +describe('a send while the first Claude start is still answering initialize', () => { + it('is held, and written once the CLI proves its start', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await host.attach(CALLER, claude.attachParams(SESSION, null)) + + await send(host, 'hello') + expect(claude.child(SESSION).calls).not.toContain('send') + + // The CLI answers: startup lands and the held message is written to the proven child. + claude.child(SESSION).answerInit() + + await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + expect(claude.children(SESSION)).toHaveLength(1) + expect(statusRows(host)).toEqual([]) + }) + + it('is rejected with the diagnostic when the CLI dies first, and restarts nothing', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await host.attach(CALLER, claude.attachParams(SESSION, null)) + const startedFence = fence(host) + + const held = await send(host, 'hello') + await failLatestStart(host, 1) + + await vi.waitFor(() => + expect(submission(host, held)).toMatchObject({ + dispatchState: 'rejected', + // Worded for the user: the red line under the composer shows it as it stands. + reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + }) + ) + expect(statusRows(host)).toEqual([ + expect.stringMatching(/stopped before it finished starting: .*not signed in \(rig\)/) + ]) + expect(fence(host)).toBe(startedFence + 1) + expect(claude.children(SESSION)).toHaveLength(1) + expect(claude.child(SESSION).calls).not.toContain('send') + }) +}) diff --git a/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts new file mode 100644 index 000000000000..0d4b49096379 --- /dev/null +++ b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts @@ -0,0 +1,351 @@ +// A send restarts a chat's Claude child and is admitted against it while it is still starting. +// When that child dies before the send's dispatch reaches the adapter, the adapter has no session +// to hold the message for, so the dispatch throws. A child that never proved its start accepted +// nothing — input is only written after initialize — so the send settles `rejected` with the +// child's own diagnostic, never as a delivery nobody can confirm, and a client that was +// subscribed the whole time receives the failure row and the rejected submission over the wire. +// Against the production runtime, adapter, record store and host, with only the CLI scripted. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope' +import type { AgentSessionSubscribeEvent } from '../../shared/agent-session-wire' +import { hostTestMessage } from '../native-chat/agent-session-wire/structured-agent-session-host-test-data' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { waitForStructuredAgentSessionRecovery } from './structured-agent-session-runtime' +import { createScriptedClaudeRuntime } from './structured-claude-scripted-runtime-test-support' + +const SESSION = 'claude-send-restart-dies-first' +const CALLER = { callerKey: 'client-1' } +const DIAGNOSTIC = 'claude stream-json exited (code 1): claude: not signed in (rig)' + +let claude = createScriptedClaudeRuntime([SESSION]) +let operations = 0 +/** The dispatch state each send was answered with, before any exit settled it. */ +const answered = new Map() + +afterEach(async () => { + vi.restoreAllMocks() + await claude.dispose() + claude = createScriptedClaudeRuntime([SESSION]) +}) + +function fence(host: StructuredAgentSessionHost): number { + return host.deps.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 +} + +function attempt(host: StructuredAgentSessionHost, text: string) { + const body = hostTestMessage(text) + return host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: `${Date.now()}-${(++operations).toString(16).padStart(32, '0')}`, + expectedRuntimeFence: fence(host), + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) +} + +async function send(host: StructuredAgentSessionHost, text: string): Promise { + const body = hostTestMessage(text) + const clientOperationId = `${Date.now()}-${(++operations).toString(16).padStart(32, '0')}` + const sent = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId, + expectedRuntimeFence: fence(host), + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + expect(sent, JSON.stringify(sent)).toMatchObject({ ok: true, replayed: false }) + if (sent.ok) { + answered.set(clientOperationId, sent.value.submission.dispatchState) + } + return clientOperationId +} + +function statusRows(host: StructuredAgentSessionHost): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) +} + +function submission(host: StructuredAgentSessionHost, clientMessageId: string) { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId) +} + +async function failLatestStart(host: StructuredAgentSessionHost, count: number): Promise { + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(count)) + claude.child(SESSION).exit(new Error(DIAGNOSTIC)) + await waitForStructuredAgentSessionRecovery() + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + ) +} + +/** The restarted child dies the instant the send's dispatch reaches the adapter. */ +function killChildAtDispatch(host: StructuredAgentSessionHost): void { + const adapter = host.deps.adapter + const dispatch = adapter.dispatch.bind(adapter) + vi.spyOn(adapter, 'dispatch').mockImplementationOnce((input) => { + claude.child(SESSION).exit(new Error(DIAGNOSTIC)) + return dispatch(input) + }) +} + +/** Everything a subscriber received, flattened to the rows and submissions it was shown. */ +function received(events: AgentSessionSubscribeEvent[]) { + const statusTexts: string[] = [] + const submissions = new Map() + const fences: number[] = [] + for (const event of events) { + if (event.type === 'end') { + continue + } + const page = event.type === 'batch' ? event.batch : event.page + for (const item of page.items) { + if (item.body.kind === 'status') { + statusTexts.push(item.body.text) + } + } + for (const entry of page.submissions) { + submissions.set(entry.clientMessageId, entry.dispatchState) + } + if (event.fence !== undefined) { + fences.push(event.fence) + } + } + return { statusTexts, submissions, fences } +} + +describe('a send whose restarted Claude child dies before the dispatch reaches the adapter', () => { + it('settles rejected with the diagnostic, keeps one failure row, and a Retry is one new attempt', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + await failLatestStart(host, 1) + const releasedFence = fence(host) + + killChildAtDispatch(host) + const sent = await send(host, 'hello?') + // The send's own answer already says it was not delivered; it does not wait for the exit. + expect(answered.get(sent)).toBe('rejected') + expect(claude.children(SESSION)).toHaveLength(2) + await waitForStructuredAgentSessionRecovery() + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + ) + + // Provably not delivered, with the cause; not "unconfirmed". + await vi.waitFor(() => + expect(submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + // Worded for the user: the red line under the composer shows it as it stands. + reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + }) + ) + expect(statusRows(host)).toEqual([ + expect.stringContaining('not signed in'), + expect.stringMatching(/stopped before it finished starting: .*not signed in \(rig\)/) + ]) + expect(fence(host)).toBe(releasedFence + 2) + expect(claude.children(SESSION)).toHaveLength(2) + expect(claude.child(SESSION).calls).not.toContain('send') + + // Retry under a new id: one restart, and once the CLI is healthy the message is written. + claude.behave(SESSION, {}) + await send(host, 'hello again') + expect(claude.children(SESSION)).toHaveLength(3) + await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + expect(claude.child(SESSION).calls.filter((call) => call === 'send')).toHaveLength(1) + expect(statusRows(host)).toHaveLength(2) + }) + + it('names the diagnostic even when the exit was fully processed before the dispatch arrived', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + await failLatestStart(host, 1) + const adapter = host.deps.adapter + const dispatch = adapter.dispatch.bind(adapter) + vi.spyOn(adapter, 'dispatch').mockImplementationOnce(async (input) => { + claude.child(SESSION).exit(new Error(DIAGNOSTIC)) + // The adapter settles and publishes the exit; the host's own settlement waits behind this send. + await new Promise((resolve) => setTimeout(resolve, 300)) + return dispatch(input) + }) + + const sent = await send(host, 'hello?') + expect(answered.get(sent)).toBe('rejected') + await waitForStructuredAgentSessionRecovery() + expect(submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + }) + expect(statusRows(host)).toHaveLength(2) + }) + + // A restart refused because its child died before it was handed over leaves one row, from the + // send, in the words any failed start uses. + it.each(['spawn', 'start-time-read'] as const)( + 'leaves one row for a restart whose child exits at %s', + async (at) => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + await failLatestStart(host, 1) + + claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: DIAGNOSTIC, at } }) + await expect(attempt(host, 'hello?')).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_owner_restart_failed' } + }) + await waitForStructuredAgentSessionRecovery() + + expect(statusRows(host)).toEqual([ + `The provider stopped before it finished starting: ${DIAGNOSTIC}.`, + `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + ]) + } + ) + + it('reaches a subscriber that was open across the restart and the exit', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + const events: AgentSessionSubscribeEvent[] = [] + const unsubscribe = host.subscribe({ + id: 'pane', + sessionId: SESSION, + emit: (event) => { + events.push(event) + } + }) + try { + await failLatestStart(host, 1) + killChildAtDispatch(host) + const sent = await send(host, 'hello?') + await waitForStructuredAgentSessionRecovery() + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + ) + + await vi.waitFor(() => { + const seen = received(events) + expect(seen.submissions.get(sent)).toBe('rejected') + expect(seen.statusTexts).toContainEqual( + expect.stringMatching(/stopped before it finished starting: .*not signed in \(rig\)/) + ) + // The subscriber ended up on the fence the exit published, not the one the restart did. + expect(seen.fences.at(-1)).toBe(fence(host)) + }) + } finally { + unsubscribe() + } + }) +}) + +describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber open throughout', () => { + const STARTUP_FAILURE = + 'The provider stopped before it finished starting: claude stream-json exited (code 1): claude: not signed in (rig).' + + /** Status rows a subscriber has been shown, one per row whatever frame carried it. */ + function shownRows(events: AgentSessionSubscribeEvent[]): Map { + const rows = new Map() + for (const event of events) { + if (event.type === 'end') { + continue + } + const page = event.type === 'batch' ? event.batch : event.page + for (const item of page.items) { + if (item.body.kind === 'status') { + rows.set(item.itemId, item.body.text) + } + } + } + return rows + } + + it('shows one row naming the cause per failed attempt, admitted or refused, and none once the CLI is fixed', async () => { + claude.behave(SESSION, { initHangs: true }) + const host = await claude.install() + const events: AgentSessionSubscribeEvent[] = [] + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + const unsubscribe = host.subscribe({ + id: 'pane', + sessionId: SESSION, + emit: (event) => { + events.push(event) + } + }) + try { + await failLatestStart(host, 1) + await vi.waitFor(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE])) + + // Send: admitted against the restarted child, which dies before starting. + killChildAtDispatch(host) + const sent = await attempt(host, 'hello?') + await waitForStructuredAgentSessionRecovery() + expect(sent).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'rejected', reason: STARTUP_FAILURE } } + }) + await vi.waitFor(() => + expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE, STARTUP_FAILURE]) + ) + + // Retry while still broken: this restart dies before its child is handed over, so the send + // is refused before admission. Still one row, saying the same thing. + claude.behave(SESSION, { + exitsDuringSpawn: { + diagnostic: 'claude stream-json exited (code 1): claude: not signed in (rig)', + at: 'start-time-read' + } + }) + await expect(attempt(host, 'hello?')).resolves.toMatchObject({ + ok: false, + refusal: { + code: 'agent_session_owner_restart_failed', + message: expect.stringMatching(/couldn't restart: .*not signed in \(rig\)/) + } + }) + await waitForStructuredAgentSessionRecovery() + await vi.waitFor(() => + expect([...shownRows(events).values()]).toEqual([ + STARTUP_FAILURE, + STARTUP_FAILURE, + STARTUP_FAILURE + ]) + ) + + // The CLI is fixed: Retry delivers and adds no row. + claude.behave(SESSION, {}) + await expect(attempt(host, 'hello?')).resolves.toMatchObject({ ok: true }) + await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + expect(shownRows(events).size).toBe(3) + } finally { + unsubscribe() + } + }) +}) diff --git a/src/main/runtime/claude-structured-session-integration.test.ts b/src/main/runtime/claude-structured-session-integration.test.ts index 02ae554b143d..e840da4944bf 100644 --- a/src/main/runtime/claude-structured-session-integration.test.ts +++ b/src/main/runtime/claude-structured-session-integration.test.ts @@ -410,24 +410,22 @@ describe('a structured Claude session over agentSession.*', () => { expect(claude.connections).toHaveLength(0) }) - it('durably returns actionable sign-in guidance when initialization has no credentials', async () => { + it('publishes, then ends the session with sign-in guidance when initialization has no credentials', async () => { claude.setInitializeAccount({ apiProvider: 'firstParty', tokenSource: 'none' }) - const params = createIntentParams() - const first = await call('agentSession.create', params) - const retry = await call('agentSession.create', params) + // The create answers once the child is spawned; the missing credentials arrive after. + await ok<{ fence: number }>('agentSession.create', createIntentParams()) + await waitForStructuredAgentSessionRecovery() - expect(first).toMatchObject({ - ok: true, - result: { - ok: false, - refusal: { - code: 'agent_session_operation_invalid', - message: expect.stringMatching(/not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s) - } - } + const guidance = itemsOf(await subscribe()).find((item) => item.body?.kind === 'status') + expect(guidance?.body).toMatchObject({ + kind: 'status', + text: expect.stringMatching( + /stopped before it finished starting: .*not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s + ) }) - expect((retry as { result: unknown }).result).toEqual((first as { result: unknown }).result) + expect(leaseOf(SESSION)).toMatchObject({ claimStatus: 'released', handoffStage: null }) + // A failed start is not auto-resumed into the same failure. expect(claude.connections).toHaveLength(1) }) @@ -440,7 +438,18 @@ describe('a structured Claude session over agentSession.*', () => { const failed = await call('agentSession.create', createIntentParams()) - expect(JSON.stringify(failed)).toContain('claude: not signed in') + // Answered once, as the refusal a replay of this operation gives, never thrown first. + expect(failed).toMatchObject({ + ok: true, + result: { + ok: false, + refusal: { + code: 'agent_session_operation_invalid', + message: expect.stringContaining('claude: not signed in'), + ownerVerdict: 'exited' + } + } + }) const lease = leaseOf(SESSION) // Latching here would refuse every later attach with agent_session_ownership_unknown, // wedging a user who only needs to sign in. @@ -455,6 +464,30 @@ describe('a structured Claude session over agentSession.*', () => { await ok<{ fence: number }>('agentSession.ensure', ensureParams(lease.runtimeFence)) }) + it('answers a create whose whole CLI tree exited as exited on the first call', async () => { + claude.setSelfExit({ + message: 'claude stream-json exited (code 1): claude: not signed in', + // The common case: the close ladder proves the root and every descendant gone. + exitVerdict: { root: 'exited', tree: 'exited' } + }) + + const failed = await call('agentSession.create', createIntentParams()) + + expect(failed).toMatchObject({ + ok: true, + result: { + ok: false, + refusal: { + code: 'agent_session_operation_invalid', + message: expect.stringContaining('claude: not signed in'), + ownerVerdict: 'exited' + } + } + }) + expect(leaseOf(SESSION)).toMatchObject({ claimStatus: 'released', handoffStage: null }) + claude.setSelfExit(null) + }) + it('keeps a session reserved when a descendant of the failed start was seen alive', async () => { claude.setSelfExit({ message: 'claude stream-json exited (code 1): claude: not signed in', diff --git a/src/main/runtime/claude-structured-startup-unanswered-control-request.test.ts b/src/main/runtime/claude-structured-startup-unanswered-control-request.test.ts new file mode 100644 index 000000000000..38d0de314d26 --- /dev/null +++ b/src/main/runtime/claude-structured-startup-unanswered-control-request.test.ts @@ -0,0 +1,251 @@ +// A Claude start has no deadline of its own, but each option write the restore replays, and +// startup's own settings read, is a control request under the ordinary request deadline. A CLI +// that answers initialize and then never answers one of those used to fault the whole session +// when that deadline fired: a start that was merely slow died with the deadline's error as its +// cause. Now the unanswered request is skipped and startup lands on the CLI's own values, while +// the saved choice stays saved for the next start to retry. Against +// the production runtime, adapter, record store and host, with only the CLI process scripted. + +import { createHash } from 'node:crypto' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope' +import { claudeSessionIdForOrcaSession } from '../claude/claude-structured-launch-resolution' +import { hostTestMessage } from '../native-chat/agent-session-wire/structured-agent-session-host-test-data' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { createScriptedClaudeRuntime } from './structured-claude-scripted-runtime-test-support' + +const SESSION = 'claude-startup-unanswered-control' +const CALLER = { callerKey: 'client-1' } +const DEADLINE_MS = 50 + +let claude = createScriptedClaudeRuntime([SESSION]) +let operations = 0 + +afterEach(async () => { + await claude.dispose() + claude = createScriptedClaudeRuntime([SESSION]) +}) + +function record(host: StructuredAgentSessionHost) { + return host.deps.store.getRecord(SESSION) +} + +function operationId(): string { + return `${Date.now()}-${(++operations).toString(16).padStart(32, '0')}` +} + +async function setOption( + host: StructuredAgentSessionHost, + key: string, + value: string +): Promise { + const changed = await host.setOption(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: operationId(), + expectedRuntimeFence: record(host)?.lease.runtimeFence ?? 0, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.setOption', + sessionId: SESSION, + fields: { key, value } + }) + }, + key, + value + }) + expect(changed, JSON.stringify(changed)).toMatchObject({ ok: true }) +} + +/** What the picker is handed: the value it shows, and which ones the CLI vouched for. */ +async function picker(host: StructuredAgentSessionHost) { + const { model, effort, confirmed } = (await host.readOptions(SESSION)).current + return { model, effort, confirmed } +} + +/** The CLI opens a turn by naming the model it is actually running. */ +function turnReportsModel(model: string): void { + claude.child(SESSION).handlers.onMessage?.({ + type: 'system', + subtype: 'init', + session_id: claudeSessionIdForOrcaSession(SESSION), + model + }) +} + +function statusRows(host: StructuredAgentSessionHost): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) +} + +async function send(host: StructuredAgentSessionHost, text: string): Promise { + const body = hostTestMessage(text) + await expect( + host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: `${Date.now()}-${(++operations).toString(16).padStart(32, '0')}`, + expectedRuntimeFence: record(host)?.lease.runtimeFence ?? 0, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + ).resolves.toMatchObject({ ok: true }) +} + +describe('a Claude start whose CLI answers initialize but not a control request', () => { + it('lands with the unanswered option write skipped instead of faulting at the deadline, and keeps the saved choice', async () => { + claude.behave(SESSION, { optionWritesHang: true, controlTimeoutMs: DEADLINE_MS }) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = await claude.install() + const saved = { model: 'sonnet', permissionMode: 'plan' } + await expect( + host.attach(CALLER, claude.attachParams(SESSION, null, { options: saved })) + ).resolves.toMatchObject({ ok: true }) + + // The restore asked; the CLI never answered; startup went on without it. + await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('set_model')) + await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('set_permission_mode')) + await vi.waitFor(() => expect(record(host)?.lease.claimStatus).toBe('live'), { + timeout: DEADLINE_MS * 40 + }) + // The live child runs on the CLI's own model; silence is not a refusal, so the saved + // choice is neither replaced by that value nor dropped, and the next start retries it. + await vi.waitFor(() => expect(record(host)?.options).toEqual({ ...saved, effort: 'high' }), { + timeout: DEADLINE_MS * 40 + }) + expect(host.deps.adapter.readOptionRestoreFailures?.(SESSION)).toEqual([]) + expect(statusRows(host)).toEqual([]) + expect(claude.children(SESSION)).toHaveLength(1) + + // The proven child takes the next message. + await send(host, 'hello') + await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + }) + + it('keeps the unanswered saved choice when the user later changes a different option', async () => { + const behavior = { optionWritesHang: true, controlTimeoutMs: DEADLINE_MS } + claude.behave(SESSION, behavior) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = await claude.install() + await expect( + host.attach(CALLER, claude.attachParams(SESSION, null, { options: { model: 'sonnet' } })) + ).resolves.toMatchObject({ ok: true }) + await vi.waitFor( + () => expect(record(host)?.options).toEqual({ model: 'sonnet', effort: 'high' }), + { + timeout: DEADLINE_MS * 40 + } + ) + + // The CLI answers again; the user sets another option on the running child. + behavior.optionWritesHang = false + await setOption(host, 'permissionMode', 'plan') + expect(record(host)?.options).toMatchObject({ model: 'sonnet', permissionMode: 'plan' }) + }) + + it('replays the unanswered saved model after a turn reports another model, another option changes, and the chat is cleared', async () => { + const clearOperation = operationId() + const replacement = `clear-${createHash('sha256') + .update(JSON.stringify([SESSION, CALLER.callerKey, clearOperation])) + .digest('hex') + .slice(0, 40)}` + claude = createScriptedClaudeRuntime([SESSION, replacement]) + const behavior = { optionWritesHang: true, controlTimeoutMs: DEADLINE_MS } + claude.behave(SESSION, behavior) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = await claude.install() + await expect( + host.attach(CALLER, claude.attachParams(SESSION, null, { options: { model: 'sonnet' } })) + ).resolves.toMatchObject({ ok: true }) + await vi.waitFor( + () => expect(record(host)?.options).toEqual({ model: 'sonnet', effort: 'high' }), + { timeout: DEADLINE_MS * 40 } + ) + // The picker offers the saved model, which nothing has vouched for yet. + expect(await picker(host)).toEqual({ model: 'sonnet', effort: 'high', confirmed: ['effort'] }) + + // A turn shows the child running the CLI's own model: the picker follows it, the record + // keeps what the user chose. + turnReportsModel('claude-sonnet-5') + expect(await picker(host)).toEqual({ + model: 'claude-sonnet-5', + effort: 'high', + confirmed: ['model', 'effort'] + }) + expect(record(host)?.options).toEqual({ model: 'sonnet', effort: 'high' }) + + behavior.optionWritesHang = false + await setOption(host, 'permissionMode', 'plan') + expect(record(host)?.options).toEqual({ model: 'sonnet', permissionMode: 'plan' }) + + const cleared = await host.conversationCommand(CALLER, { + command: 'clear', + envelope: { + sessionId: SESSION, + clientOperationId: clearOperation, + expectedRuntimeFence: record(host)?.lease.runtimeFence ?? 0, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.conversationCommand', + sessionId: SESSION, + fields: { command: 'clear' } + }) + } + }) + expect(cleared, JSON.stringify(cleared)).toMatchObject({ + ok: true, + value: { replacementSessionId: replacement } + }) + // The cleared chat's start replays the saved model rather than the one the turn reported. + await vi.waitFor(() => expect(claude.child(replacement).calls).toContain('set_model')) + await vi.waitFor(() => + expect(host.deps.store.getRecord(replacement)?.options).toEqual({ + model: 'sonnet', + effort: 'high', + permissionMode: 'plan' + }) + ) + expect(record(host)?.options).toEqual({ model: 'sonnet', permissionMode: 'plan' }) + }) + + it('replaces the unanswered saved model with the one the user then sets', async () => { + const behavior = { optionWritesHang: true, controlTimeoutMs: DEADLINE_MS } + claude.behave(SESSION, behavior) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = await claude.install() + await expect( + host.attach(CALLER, claude.attachParams(SESSION, null, { options: { model: 'sonnet' } })) + ).resolves.toMatchObject({ ok: true }) + // The record holds the saved model from creation; only the start's own report (effort) says + // startup finished, and an option write before then is refused as still starting. + await vi.waitFor( + () => expect(record(host)?.options).toEqual({ model: 'sonnet', effort: 'high' }), + { timeout: DEADLINE_MS * 40 } + ) + turnReportsModel('claude-sonnet-5') + + behavior.optionWritesHang = false + await setOption(host, 'model', 'opus') + expect(record(host)?.options).toEqual({ model: 'opus' }) + expect((await picker(host)).model).toBe('opus') + }) + + it("lands with effort unknown when startup's own settings read goes unanswered", async () => { + claude.behave(SESSION, { startupSettingsReadHangs: true, controlTimeoutMs: DEADLINE_MS }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + + await vi.waitFor(() => expect(record(host)?.options).toEqual({ model: 'claude-sonnet-5' }), { + timeout: DEADLINE_MS * 40 + }) + expect(record(host)?.lease.claimStatus).toBe('live') + expect(statusRows(host)).toEqual([]) + expect(claude.children(SESSION)).toHaveLength(1) + }) +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts index da34108604ab..c3099d843ced 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts @@ -40,6 +40,7 @@ let host: StructuredAgentSessionHost let runtime: OrcaRuntimeService let dispatcher: RpcDispatcher let closeSession: Mock> +let acquire: Mock let requests = 0 let structuredNativeChatEnabled = true @@ -60,20 +61,26 @@ beforeEach(async () => { requests = 0 structuredNativeChatEnabled = true closeSession = vi.fn(async () => true) + acquire = vi.fn(async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } + })) store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) host = new StructuredAgentSessionHost({ store, adapter: { - acquire: async ({ fence, spawnToken }) => ({ - process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, - link: { - linkId: `link-${fence}`, - handle: { provider: 'codex', threadId: THREAD }, - origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', - mintedAtFence: fence, - observedAt: NOW - } - }), + acquire, + // A failed acquisition is proven gone, as the real adapters prove it; without this an + // acquire that throws leaves an unverifiable owner nothing may replace. + releaseAcquisition: vi.fn(async () => true), closeSession, dispatch: async () => ({ state: 'rejected', reason: 'unused' }), cancelTurn: async () => ({ cancelled: false }), @@ -162,106 +169,77 @@ describe('a client that holds a session', () => { }) describe('a client that disappears without cleanup', () => { - it('releases a late child after its same-ID replacement refuses the stale fence', async () => { + it('shares one child with a same-ID replacement that arrives while the first hold resumes', async () => { await host.close(SESSION) await host.restoreReadableSessions() closeSession.mockClear() - const firstEntered = Promise.withResolvers() - const firstGate = Promise.withResolvers() - const replacementEntered = Promise.withResolvers() - const replacementGate = Promise.withResolvers() - const attach = host.attach.bind(host) - const attachSpy = vi - .spyOn(host, 'attach') - .mockImplementationOnce(async (...args) => { - firstEntered.resolve() - await firstGate.promise - return attach(...args) - }) - .mockImplementationOnce(async (...args) => { - replacementEntered.resolve() - await replacementGate.promise - return attach(...args) - }) + acquire.mockClear() + const entered = Promise.withResolvers() + const gate = Promise.withResolvers() + const spawnChild = acquire.getMockImplementation()! + acquire.mockImplementationOnce(async (input) => { + entered.resolve() + await gate.promise + return spawnChild(input) + }) try { const params = { sessionId: SESSION, holderId: 'same-chat' } const first = call('agentSession.hold', params) - await firstEntered.promise + await entered.promise + // Re-registering the cleanup id released the first hold; the replacement waits its turn + // behind the first hold's attach and finds the child it made. const replacement = call('agentSession.hold', params) - await replacementEntered.promise - firstGate.resolve() + gate.resolve() + expect(await first).toMatchObject({ ok: true }) + expect(await replacement).toMatchObject({ ok: true }) + expect(acquire).toHaveBeenCalledOnce() expect(host.isHeld(SESSION)).toBe(true) + await new Promise((resolve) => setTimeout(resolve, GRACE_MS * 4)) expect(closeSession).not.toHaveBeenCalled() - replacementGate.resolve() - expect(await replacement).toMatchObject({ - ok: false, - error: { code: 'agent_session_checkpoint_stale' } - }) - expect(host.isHeld(SESSION)).toBe(false) + runtime.cleanupSubscriptionsForConnection(CONNECTION) await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) expect(closeSession).toHaveBeenCalledExactlyOnceWith(SESSION) } finally { - firstGate.resolve() - replacementGate.resolve() - attachSpy.mockRestore() + gate.resolve() } }) - it.each([false, true])( - 'keeps replacement hold and cleanup after an old request fails (replacement finished=%s)', - async (replacementFinished) => { - await host.close(SESSION) - await host.restoreReadableSessions() - closeSession.mockClear() - const firstEntered = Promise.withResolvers() - const firstGate = Promise.withResolvers() - const replacementEntered = Promise.withResolvers() - const replacementGate = Promise.withResolvers() - const attach = host.attach.bind(host) - const attachSpy = vi - .spyOn(host, 'attach') - .mockImplementationOnce(async () => { - firstEntered.resolve() - await firstGate.promise - throw new Error('old acquisition failed') - }) - .mockImplementationOnce(async (...args) => { - replacementEntered.resolve() - await replacementGate.promise - return attach(...args) - }) - try { - const params = { sessionId: SESSION, holderId: 'same-chat' } - const first = call('agentSession.hold', params) - await firstEntered.promise - const replacement = call('agentSession.hold', params) - await replacementEntered.promise - if (replacementFinished) { - replacementGate.resolve() - expect(await replacement).toMatchObject({ ok: true }) - } - - firstGate.resolve() - expect(await first).toMatchObject({ ok: false }) - expect(host.isHeld(SESSION)).toBe(true) - replacementGate.resolve() - expect(await replacement).toMatchObject({ ok: true }) - await new Promise((resolve) => setTimeout(resolve, GRACE_MS * 4)) - expect(host.hasSession(SESSION)).toBe(true) - expect(closeSession).not.toHaveBeenCalled() - - runtime.cleanupSubscriptionsForConnection(CONNECTION) - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledExactlyOnceWith(SESSION) - } finally { - firstGate.resolve() - replacementGate.resolve() - attachSpy.mockRestore() - } + it('lets a same-ID replacement make its own attempt when the first hold fails to acquire', async () => { + await host.close(SESSION) + await host.restoreReadableSessions() + closeSession.mockClear() + acquire.mockClear() + const entered = Promise.withResolvers() + const gate = Promise.withResolvers() + acquire.mockImplementationOnce(async () => { + entered.resolve() + await gate.promise + throw new Error('acquisition failed') + }) + try { + const params = { sessionId: SESSION, holderId: 'same-chat' } + const first = call('agentSession.hold', params) + await entered.promise + const replacement = call('agentSession.hold', params) + gate.resolve() + + expect(await first).toMatchObject({ ok: false }) + // One attempt per hold: the replacement's own succeeds, and the holder it re-took stands. + const replaced = await replacement + expect(replaced, JSON.stringify(replaced)).toMatchObject({ ok: true }) + expect(acquire).toHaveBeenCalledTimes(2) + expect(host.isHeld(SESSION)).toBe(true) + expect(closeSession).not.toHaveBeenCalled() + + runtime.cleanupSubscriptionsForConnection(CONNECTION) + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) + expect(closeSession).toHaveBeenCalledExactlyOnceWith(SESSION) + } finally { + gate.resolve() } - ) + }) it('still releases the session when its transport closes', async () => { await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) diff --git a/src/main/runtime/structured-agent-session-lifecycle-delivery.ts b/src/main/runtime/structured-agent-session-lifecycle-delivery.ts new file mode 100644 index 000000000000..961c4d50aa84 --- /dev/null +++ b/src/main/runtime/structured-agent-session-lifecycle-delivery.ts @@ -0,0 +1,59 @@ +// How provider lifecycle events reach the host, and how teardown knows every one has landed. +// +// Exit recovery runs on one chain so teardown can drain it: exit callbacks arrive from child +// process tasks, and a fire-and-forget one could otherwise append after the host flushed and +// removed its journal directory. That chain orders nothing across sessions, and a recovery on it +// can run a whole reacquisition, so `started` stays off it: it takes only its own session's +// serialized step, and is tracked here so the same drain still waits for it. + +import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' + +export function createStructuredAgentSessionLifecycleDelivery(input: { + handle: (event: StructuredAgentSessionLifecycleEvent) => Promise | undefined + onError?: (input: { scope: string; error: unknown }) => void + /** Exits the adapter has observed but not yet published. */ + drainObservedExits: () => Promise +}): { + deliver: (event: StructuredAgentSessionLifecycleEvent) => void + /** Resolves once every observed exit has published and everything delivered has settled. */ + drain: () => Promise +} { + let recoveryChain = Promise.resolve() + const settlingStarts = new Set>() + const settle = async (event: StructuredAgentSessionLifecycleEvent): Promise => { + try { + await input.handle(event) + } catch (error) { + const scope = event.type === 'started' ? 'started' : 'exit' + input.onError?.({ scope: `structured-agent-session-${scope}:${event.sessionId}`, error }) + } + } + return { + deliver: (event) => { + if (event.type === 'started') { + // Called now, so the step is queued on its session ahead of any later exit of that child. + const settling = settle(event) + settlingStarts.add(settling) + void settling.finally(() => settlingStarts.delete(settling)) + return + } + recoveryChain = recoveryChain.then(() => settle(event)) + }, + drain: async () => { + // A recovery may synchronously trigger another exit while it is reacquiring, so observe + // until nothing new arrives. + for (;;) { + await input.drainObservedExits() + const observed = recoveryChain + await observed + if (settlingStarts.size > 0) { + await Promise.all(settlingStarts) + continue + } + if (observed === recoveryChain) { + return + } + } + } + } +} diff --git a/src/main/runtime/structured-agent-session-runtime-provider-started.test.ts b/src/main/runtime/structured-agent-session-runtime-provider-started.test.ts new file mode 100644 index 000000000000..602507ef4d7e --- /dev/null +++ b/src/main/runtime/structured-agent-session-runtime-provider-started.test.ts @@ -0,0 +1,121 @@ +// A Claude child proving its start must not wait on another session's exit recovery, and must +// not make that recovery, or the session's own serialized operations, wait on the CLI: the host +// records what the child proved from what the adapter already holds. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { waitForStructuredAgentSessionRecovery } from './structured-agent-session-runtime' +import { createScriptedClaudeRuntime } from './structured-claude-scripted-runtime-test-support' + +const STALLED = 'claude-started-stalled' +const HEALTHY = 'claude-started-healthy' +const CALLER = { callerKey: 'client-1' } + +let claude = createScriptedClaudeRuntime([STALLED, HEALTHY]) + +afterEach(async () => { + await claude.dispose() + claude = createScriptedClaudeRuntime([STALLED, HEALTHY]) +}) + +describe('a Claude child proving its start', () => { + it("never holds another session's exit recovery, or its own close, on a CLI read", async () => { + claude.behave(STALLED, { stallsControlReads: true }) + const host = await claude.install() + + await expect(host.attach(CALLER, claude.attachParams(HEALTHY, null))).resolves.toMatchObject({ + ok: true + }) + await waitForStructuredAgentSessionRecovery() + + // This child answers startup, then never answers another control read. + await expect(host.attach(CALLER, claude.attachParams(STALLED, null))).resolves.toMatchObject({ + ok: true + }) + await vi.waitFor(() => expect(claude.child(STALLED).calls).toContain('get_settings')) + // Its `started` reaches the shared chain ahead of the exit below. + await new Promise((resolve) => setImmediate(resolve)) + + claude.child(HEALTHY).exit(new Error('claude stream-json exited (code 1): crashed')) + await vi.waitFor(() => + expect(host.deps.store.getRecord(HEALTHY)?.lease.claimStatus).toBe('released') + ) + + // What the child proved is still recorded, from the startup it already answered. + await vi.waitFor(() => + expect(host.deps.store.getRecord(STALLED)?.options).toEqual({ + model: 'claude-sonnet-5', + effort: 'high' + }) + ) + expect(claude.child(STALLED).calls).toEqual(['get_settings']) + + let closed = false + void host.close(STALLED).then(() => { + closed = true + }) + await vi.waitFor(() => expect(closed).toBe(true)) + }) + + it("flips to ready while another session's exit recovery is still acquiring", async () => { + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(HEALTHY, null))).resolves.toMatchObject({ + ok: true + }) + await host.hold(HEALTHY, 'surface-1') + await waitForStructuredAgentSessionRecovery() + + // Its exit recovery reacquires, and that spawn never returns. The exit hands the lease back + // and the restart queued behind it reserves it again at once, so `released` is not a state a + // poll can count on seeing; `reserved` with the spawn hanging is what "still acquiring" is. + claude.behave(HEALTHY, { spawnHangs: true }) + claude.child(HEALTHY).exit(new Error('claude stream-json exited (code 1): crashed')) + await vi.waitFor(() => + expect(host.deps.store.getRecord(HEALTHY)?.lease.claimStatus).toBe('reserved') + ) + expect(claude.children(HEALTHY)).toHaveLength(1) + + await expect(host.attach(CALLER, claude.attachParams(STALLED, null))).resolves.toMatchObject({ + ok: true + }) + await vi.waitFor(() => + expect(host.deps.store.getRecord(STALLED)?.options).toEqual({ + model: 'claude-sonnet-5', + effort: 'high' + }) + ) + // The other recovery is still where it was: reserved, with no child yet. + expect(host.deps.store.getRecord(HEALTHY)?.lease.claimStatus).toBe('reserved') + expect(claude.children(HEALTHY)).toHaveLength(1) + }) + + it('is drained by the runtime before teardown proceeds', async () => { + const host = await claude.install() + const store = host.deps.store + const replaceSessionOptions = store.replaceSessionOptions.bind(store) + let landWrite = (): void => {} + const writeHeld = new Promise((resolve) => { + landWrite = resolve + }) + let writing = false + vi.spyOn(store, 'replaceSessionOptions').mockImplementation(async (input) => { + writing = true + await writeHeld + return replaceSessionOptions(input) + }) + await expect(host.attach(CALLER, claude.attachParams(HEALTHY, null))).resolves.toMatchObject({ + ok: true + }) + await vi.waitFor(() => expect(writing).toBe(true)) + + let drained = false + const recovery = waitForStructuredAgentSessionRecovery().then(() => { + drained = true + }) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(drained).toBe(false) + + landWrite() + await recovery + expect(store.getRecord(HEALTHY)?.options).toEqual({ model: 'claude-sonnet-5', effort: 'high' }) + }) +}) diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index 4497290dc346..0ec7996bb91b 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -50,6 +50,7 @@ import { createStructuredAgentEnvironmentResolvers } from './structured-agent-sh import { recordAgentSessionProviderHandle } from './agent-session-provider-handle-transition' import type { ClaudeStructuredAuthPolicy } from '../claude-accounts/claude-structured-auth-policy' import { createStructuredClaudeRuntimeAdapter } from './structured-claude-runtime-adapter' +import { createStructuredAgentSessionLifecycleDelivery } from './structured-agent-session-lifecycle-delivery' /** Sibling of the journal tree rather than inside it: one file adjudicates every * session's lease, while a journal is per session. */ @@ -210,7 +211,13 @@ async function install(deps: StructuredAgentSessionRuntimeDeps): Promise host?.handleAdapterEvent(event), + ...(deps.onError ? { onError: deps.onError } : {}), + // Claude publishes an observed exit only after its close ladder and transcript write; Codex + // publishes inside its own exit callback and needs nothing. + drainObservedExits: () => claude.drainObservedExits() + }) const onDispatchSettledLate = ( settlement: Parameters[0] ): void => { @@ -250,19 +257,9 @@ async function install(deps: StructuredAgentSessionRuntimeDeps): Promise { - if (event.type !== 'ended' || !('cause' in event) || event.cause !== 'unexpected-exit') { - return + if (event.type === 'ended' && 'cause' in event && event.cause === 'unexpected-exit') { + lifecycle.deliver(event) } - // Serialize recovery with teardown. Exit callbacks arrive from child - // process tasks, so a fire-and-forget callback can otherwise append - // after the host has flushed and its journal directory is removed. - recoveryChain = recoveryChain.then(async () => { - try { - await host?.handleAdapterEvent(event) - } catch (error) { - deps.onError?.({ scope: `structured-agent-session-exit:${event.sessionId}`, error }) - } - }) } }) const claude = createStructuredClaudeRuntimeAdapter({ @@ -283,15 +280,7 @@ async function install(deps: StructuredAgentSessionRuntimeDeps): Promise { - recoveryChain = recoveryChain.then(async () => { - try { - await host?.handleAdapterEvent(event) - } catch (error) { - deps.onError?.({ scope: `structured-agent-session-exit:${event.sessionId}`, error }) - } - }) - }, + onLifecycleEvent: (event) => lifecycle.deliver(event), onBackgroundTasksChanged: (sessionId, state) => host?.publishBackgroundTaskState(sessionId, state), onDispatchSettledLate, @@ -332,21 +321,7 @@ async function install(deps: StructuredAgentSessionRuntimeDeps): Promise { - // A recovery may synchronously trigger another exit while it is - // reacquiring. Observe until the chain stops growing. - for (;;) { - // Claude reaches the chain only once its close ladder and transcript - // write publish the exit, so an observed death is not yet a chained - // one. Codex publishes inside its own exit callback and needs nothing. - await claude.drainObservedExits() - const observed = recoveryChain - await observed - if (observed === recoveryChain) { - return - } - } - } + waitForRecovery: lifecycle.drain } } catch (error) { agentSessionPtyWriteGate.detachRecordLookup() diff --git a/src/main/runtime/structured-claude-pending-rewind.test.ts b/src/main/runtime/structured-claude-pending-rewind.test.ts index c7774ffa7fd1..38909e1a9bd4 100644 --- a/src/main/runtime/structured-claude-pending-rewind.test.ts +++ b/src/main/runtime/structured-claude-pending-rewind.test.ts @@ -2,7 +2,7 @@ // RPC leaves nothing behind, and that a pending rewind persisted by an older build never strands // the chat: the next attach resumes by session id and settles the rewind as refused. -import { mkdtemp, rm } from 'node:fs/promises' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -67,6 +67,10 @@ const fence = (): number => store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFen /** What an older build left behind: an admitted rewind whose outcome was never recorded. */ async function seedPendingRewind(phase: 'prepared' | 'provider-succeeded') { const request = rewindParams(fence()) + // A rewind had a turn to target, so Claude had written the transcript a resume continues. + const projects = join(directory, 'claude-home', 'projects', 'workspace') + await mkdir(projects, { recursive: true }) + await writeFile(join(projects, `${PROVIDER_SESSION_ID}.jsonl`), '') await store.admitMutationOperation({ callerKey: caller.callerKey, envelope: request.envelope, @@ -113,7 +117,7 @@ beforeEach(async () => { resolveClaudeAuthPolicy: () => ({ stripAuthEnv: false }), openClaudeConnection: claude.openConnection, readProcessStartTime: async () => HOST_TEST_NOW, - onUnexpectedExit: () => {} + onLifecycleEvent: () => {} }) host = new StructuredAgentSessionHost({ store, diff --git a/src/main/runtime/structured-claude-runtime-adapter.ts b/src/main/runtime/structured-claude-runtime-adapter.ts index b4aff275ace6..6e84eed7eb5c 100644 --- a/src/main/runtime/structured-claude-runtime-adapter.ts +++ b/src/main/runtime/structured-claude-runtime-adapter.ts @@ -10,6 +10,7 @@ import { } from '../claude/claude-structured-session-adapter' import { claudeProviderHandleLink } from '../claude/claude-structured-owner-identity' import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { ClaudeStructuredSessionEvent } from '../claude/claude-structured-session-state' import { recordAgentSessionProviderHandle, reviseAgentSessionClaudeResumePoint @@ -32,7 +33,7 @@ export type StructuredClaudeRuntimeAdapterDeps = { readClaudeManagedAccountGate?: () => ClaudeManagedAccountGateSettings | null openClaudeConnection?: ClaudeStructuredSessionAdapterDeps['openConnection'] readProcessStartTime?: ClaudeStructuredSessionAdapterDeps['readProcessStartTime'] - onUnexpectedExit: (event: StructuredAgentSessionLifecycleEvent) => void + onLifecycleEvent: (event: StructuredAgentSessionLifecycleEvent) => void onBackgroundTasksChanged?: ( sessionId: string, state: AgentSessionBackgroundTaskState | null @@ -40,6 +41,35 @@ export type StructuredClaudeRuntimeAdapterDeps = { onDispatchSettledLate?: ClaudeStructuredSessionAdapterDeps['onDispatchSettledLate'] } +/** The adapter events the host's lifecycle handler consumes, in the host's vocabulary. */ +export function structuredClaudeLifecycleEvent( + event: ClaudeStructuredSessionEvent +): StructuredAgentSessionLifecycleEvent | null { + if (event.type === 'started') { + return event + } + if ( + event.type === 'ended' && + event.cause === 'unexpected-exit' && + event.fence !== undefined && + event.acquisitionGeneration + ) { + return { + type: 'ended', + sessionId: event.sessionId, + reason: event.reason, + cause: event.cause, + fence: event.fence, + acquisitionGeneration: event.acquisitionGeneration, + ...(event.settlementRetryRequired + ? { settlementRetryRequired: event.settlementRetryRequired } + : {}), + ...(event.startupUnproven ? { startupUnproven: event.startupUnproven } : {}) + } + } + return null +} + export function createStructuredClaudeRuntimeAdapter( deps: StructuredClaudeRuntimeAdapterDeps ): ClaudeStructuredSessionAdapter { @@ -91,23 +121,9 @@ export function createStructuredClaudeRuntimeAdapter( ) }, onEvent: (event) => { - if ( - event.type === 'ended' && - event.cause === 'unexpected-exit' && - event.fence !== undefined && - event.acquisitionGeneration - ) { - deps.onUnexpectedExit({ - type: 'ended', - sessionId: event.sessionId, - reason: event.reason, - cause: event.cause, - fence: event.fence, - acquisitionGeneration: event.acquisitionGeneration, - ...(event.settlementRetryRequired - ? { settlementRetryRequired: event.settlementRetryRequired } - : {}) - }) + const lifecycle = structuredClaudeLifecycleEvent(event) + if (lifecycle) { + deps.onLifecycleEvent(lifecycle) } }, ...(deps.onBackgroundTasksChanged diff --git a/src/main/runtime/structured-claude-scripted-runtime-test-support.ts b/src/main/runtime/structured-claude-scripted-runtime-test-support.ts new file mode 100644 index 000000000000..af1bc9a4af71 --- /dev/null +++ b/src/main/runtime/structured-claude-scripted-runtime-test-support.ts @@ -0,0 +1,252 @@ +// A structured-session runtime whose Claude children are scripted: the production runtime, +// adapter, record store and host, with only the CLI process replaced. + +import { mkdir, mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { + ClaudeStreamJsonConnection, + ClaudeStreamJsonConnectionHandlers, + ClaudeStreamJsonLaunch, + openClaudeStreamJsonConnection +} from '../claude/claude-stream-json-connection' +import { runClaudeControl } from '../claude/claude-agent-sdk-control-requests' +import { claudeSessionIdForOrcaSession } from '../claude/claude-structured-launch-resolution' +import type { AgentSessionAttachParams } from '../native-chat/agent-session-wire/structured-agent-session-attach' +import { hostTestAttachParams } from '../native-chat/agent-session-wire/structured-agent-session-host-test-data' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { + ensureStructuredAgentSessionHost, + stopStructuredAgentSessionRuntime +} from './structured-agent-session-runtime' + +export type ScriptedClaudeBehavior = { + /** Initialize never answers; only the child's exit settles it. */ + initHangs?: boolean + /** Every control read after startup's own settings read waits for `releaseStalls`. */ + stallsControlReads?: boolean + /** The spawn itself waits for `releaseStalls`, holding its acquisition open. */ + spawnHangs?: boolean + /** The CLI exits with this diagnostic before its spawn returns, or while its start time is read. */ + exitsDuringSpawn?: { diagnostic: string; at: 'spawn' | 'start-time-read' } + /** Closing cannot prove the descendant tree gone, as when it was never snapshottable. */ + closeUnproven?: boolean + /** Control requests run under this deadline, the way production's run under the default one. */ + controlTimeoutMs?: number + /** Every option write (set_model, set_permission_mode, apply_flag_settings) goes unanswered. */ + optionWritesHang?: boolean + /** Startup's own settings read goes unanswered. */ + startupSettingsReadHangs?: boolean +} + +export type ScriptedClaudeChild = { + sessionId: string + launch: ClaudeStreamJsonLaunch + calls: string[] + handlers: ClaudeStreamJsonConnectionHandlers + connection: Omit & { + closed: boolean + exitVerdict: ClaudeStreamJsonConnection['exitVerdict'] + } + /** The CLI exits on its own: its root is gone, its tree unverifiable. */ + exit: (error: Error) => void + /** The CLI answers initialize now; only meaningful under `initHangs`. */ + answerInit: () => void +} + +export function createScriptedClaudeRuntime(sessionIds: readonly string[]) { + const children: ScriptedClaudeChild[] = [] + const behaviors = new Map() + let releaseStalls = (): void => {} + const stall = new Promise((resolve) => { + releaseStalls = resolve + }) + let root: string | null = null + let operations = 0 + + const openConnection: typeof openClaudeStreamJsonConnection = async (launch, handlers = {}) => { + const providerSessionId = String(launch.options.sessionId ?? launch.options.resume) + const sessionId = sessionIds.find( + (candidate) => claudeSessionIdForOrcaSession(candidate) === providerSessionId + ) + if (!sessionId) { + throw new Error(`no scripted Claude session for ${providerSessionId}`) + } + const behavior = behaviors.get(sessionId) ?? {} + if (behavior.spawnHangs) { + await stall + } + let failInit = (_error: Error): void => {} + let answerInit = (): void => {} + const answer = (value: T, startup: boolean): Promise => + behavior.stallsControlReads && !startup ? stall.then(() => value) : Promise.resolve(value) + // Untimed unless the behavior sets a deadline; a hang then settles only on the child's exit. + const control = (subtype: string, run: () => Promise): Promise => + runClaudeControl(subtype, run, behavior.controlTimeoutMs ?? null) + const never = (): Promise => new Promise(() => {}) + const optionWrite = (subtype: string): Promise => { + child.calls.push(subtype) + return control(subtype, () => (behavior.optionWritesHang ? never() : Promise.resolve())) + } + let settingsReads = 0 + const child: ScriptedClaudeChild = { + sessionId, + launch, + calls: [], + handlers, + exit: (error) => { + child.connection.exitVerdict = { root: 'exited', tree: 'unverifiable' } + failInit(error) + handlers.onExit?.(error) + }, + answerInit: () => answerInit(), + connection: { + pid: 5000 + children.length, + closed: false, + exitVerdict: { root: 'live', tree: 'unverifiable' }, + initializationResult: () => { + const initialized = { models: [{ value: 'sonnet', displayName: 'Sonnet' }] } + const announce = (): void => + handlers.onMessage?.({ + type: 'system', + subtype: 'init', + session_id: providerSessionId, + model: 'claude-sonnet-5', + apiKeySource: 'none' + }) + if (behavior.initHangs) { + return new Promise((resolve, reject) => { + failInit = reject + answerInit = () => { + announce() + resolve(initialized) + } + }) + } + announce() + return Promise.resolve(initialized) + }, + getContextUsage: () => { + child.calls.push('get_context_usage') + return Promise.resolve({}) + }, + getSettings: () => { + child.calls.push('get_settings') + settingsReads += 1 + const startup = settingsReads === 1 + return control('get_settings', () => + behavior.startupSettingsReadHangs && startup + ? never() + : answer({ effective: { effortLevel: 'high' } }, startup) + ) + }, + supportedModels: () => { + child.calls.push('list_models') + return control('list_models', () => + answer( + [ + { value: 'sonnet', displayName: 'Sonnet' }, + { value: 'opus', displayName: 'Opus' } + ], + false + ) + ) + }, + setModel: () => optionWrite('set_model'), + setPermissionMode: () => optionWrite('set_permission_mode'), + applyFlagSettings: () => optionWrite('apply_flag_settings'), + interrupt: async () => undefined, + cancelAsyncMessage: async () => {}, + stopTask: async () => {}, + send: async () => { + child.calls.push('send') + }, + close: async () => { + child.connection.closed = true + return !behavior.closeUnproven + } + } + } + children.push(child) + if (behavior.exitsDuringSpawn?.at === 'spawn') { + exitAtSpawn(child, behavior.exitsDuringSpawn.diagnostic) + } + return child.connection + } + const exitAtSpawn = (child: ScriptedClaudeChild, diagnostic: string): void => { + child.connection.closed = true + child.exit(new Error(diagnostic)) + } + /** A pid whose process is gone has no start time to read. */ + const readProcessStartTime = async (pid: number): Promise => { + const child = children.find((entry) => entry.connection.pid === pid) + const exits = child ? behaviors.get(child.sessionId)?.exitsDuringSpawn : undefined + if (child && exits?.at === 'start-time-read' && !child.connection.closed) { + exitAtSpawn(child, exits.diagnostic) + return pid * 10 + } + return child?.connection.exitVerdict.root === 'exited' ? null : pid * 10 + } + + return { + behave: (sessionId: string, behavior: ScriptedClaudeBehavior): void => { + behaviors.set(sessionId, behavior) + }, + /** The latest child spawned for `sessionId`. */ + child: (sessionId: string): ScriptedClaudeChild => { + const found = children.findLast((entry) => entry.sessionId === sessionId) + if (!found) { + throw new Error(`no Claude child for ${sessionId}`) + } + return found + }, + children: (sessionId: string): ScriptedClaudeChild[] => + children.filter((entry) => entry.sessionId === sessionId), + install: async (): Promise => { + root = await mkdtemp(join(tmpdir(), 'orca-scripted-claude-runtime-')) + await mkdir(join(root, 'claude-home'), { recursive: true }) + const directory = root + return ensureStructuredAgentSessionHost({ + stateDirectory: directory, + hostId: 'local', + claimKeyId: 'key-1', + resolveWorkspacePath: async () => directory, + resolveClaudeCommand: () => '/usr/local/bin/claude', + resolveClaudeAuthPolicy: () => ({ stripAuthEnv: false }), + openClaudeConnection: openConnection, + readProcessStartTime + }) + }, + attachParams: ( + sessionId: string, + expectedRuntimeFence: number | null, + overrides: Partial = {} + ) => + hostTestAttachParams(expectedRuntimeFence, { + envelope: { + sessionId, + // The runtime's own clock admits operation ids, so these are minted against it. + clientOperationId: `${Date.now()}-${(++operations).toString(16).padStart(32, '0')}`, + expectedRuntimeFence, + payloadFingerprint: '' + }, + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root ?? '', 'claude-home') }, + providerHandle: { + kind: 'claude', + sessionId: claudeSessionIdForOrcaSession(sessionId), + leafUuid: null + }, + ...overrides + }), + dispose: async (): Promise => { + releaseStalls() + await stopStructuredAgentSessionRuntime() + if (root) { + await rm(root, { recursive: true, force: true }) + root = null + } + } + } +} diff --git a/src/renderer/src/components/native-chat/NativeChatLaunchRetry.tsx b/src/renderer/src/components/native-chat/NativeChatLaunchRetry.tsx index 23b52615ee97..1a4204a326fa 100644 --- a/src/renderer/src/components/native-chat/NativeChatLaunchRetry.tsx +++ b/src/renderer/src/components/native-chat/NativeChatLaunchRetry.tsx @@ -5,9 +5,11 @@ import type { StructuredAgentSessionLaunchLifecycle } from '@/lib/structured-age export function NativeChatLaunchRetry({ lifecycle, + failureReason = null, onRetry }: { lifecycle: StructuredAgentSessionLaunchLifecycle | null + failureReason?: string | null onRetry: () => void }): React.JSX.Element | null { if (lifecycle !== 'failed' && lifecycle !== 'visibility-unknown') { @@ -25,7 +27,10 @@ export function NativeChatLaunchRetry({ ) return (
- {message} + + {message} + {lifecycle === 'failed' && failureReason ? ` ${failureReason}` : null} +