From 11009c513462ac312431fcc1c483c23d4adc6e73 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:31:15 -0700 Subject: [PATCH 01/22] fix(native-chat): a child's root exit is reported even during its close, and bookkeeping after it never reads as unproven - Both connections report the root process's exit once, with `expected` set when a close had begun. A close that came back unproven and whose root exits later is finished by the adapter, and its end reaches the host like any other. - A Claude close whose resume-point write fails after the exit was proven, and a Codex close whose terminal row is refused, now end the session and report the failure, instead of keeping a dead child indexed as if its exit were unproven. - A Codex close whose forced tree kill can't prove the descendants gone but saw the root exit reports the descendants and counts the root exit. - Every child exit with an identity, expected or not, is forwarded to the host. --- ...laude-stream-json-connection-close.test.ts | 33 ++++++++++++++++ .../claude/claude-stream-json-connection.ts | 8 ++-- .../claude-structured-requested-stop.test.ts | 3 +- .../claude-structured-session-acquisition.ts | 18 ++++----- .../claude-structured-session-adapter.ts | 14 ++++++- .../claude-structured-session-close.test.ts | 11 ++++-- .../claude/claude-structured-session-close.ts | 32 +++++++-------- ...claude-structured-session-recovery.test.ts | 16 +++++--- .../claude/claude-structured-session-state.ts | 2 + .../codex-app-server-connection-types.ts | 4 +- .../codex/codex-app-server-connection.test.ts | 9 +++-- src/main/codex/codex-app-server-connection.ts | 18 ++++++--- .../codex-requested-close-turn-timing.test.ts | 39 +++---------------- .../codex/codex-structured-session-acquire.ts | 4 +- .../codex-structured-session-close.test.ts | 8 ++-- .../codex/codex-structured-session-close.ts | 23 +++-------- .../codex-structured-session-teardown.ts | 3 +- ...tured-agent-session-provider-exit-proof.ts | 16 ++++++-- .../structured-agent-session-runtime.ts | 3 +- .../structured-claude-runtime-adapter.ts | 3 +- 20 files changed, 153 insertions(+), 114 deletions(-) diff --git a/src/main/claude/claude-stream-json-connection-close.test.ts b/src/main/claude/claude-stream-json-connection-close.test.ts index 22076f3600ea..f5bbc35d2698 100644 --- a/src/main/claude/claude-stream-json-connection-close.test.ts +++ b/src/main/claude/claude-stream-json-connection-close.test.ts @@ -162,6 +162,39 @@ describe('Claude stream-json close ordering', () => { expect(next).toHaveBeenCalledOnce() }) + it('reports a root that exits after its close came back unproven as that close ending', async () => { + mocks.refresh.mockReset() + mocks.proveClaudeChildExit.mockReset() + mocks.refresh.mockResolvedValue(undefined) + mocks.proveClaudeChildExit.mockResolvedValue(false) + const child = fakeChild() + const next = vi.fn(() => new Promise>>(() => {})) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This injected query exercises only the async iterator used by the connection. + const queryImpl = ((params: Parameters[0]) => { + params.options?.spawnClaudeCodeProcess?.({ + command: 'claude', + args: [], + env: {}, + signal: new AbortController().signal + }) + return { + [Symbol.asyncIterator]: () => ({ next }) + } + }) as unknown as typeof query + const exits: (boolean | undefined)[] = [] + const connection = await openClaudeStreamJsonConnection( + { pathToClaudeCodeExecutable: 'claude', options: {}, cwd: '/work/repo' }, + { onExit: (_error, exit) => exits.push(exit?.expected) }, + () => child, + queryImpl + ) + await expect(connection.close()).resolves.toBe(false) + + child.emit('exit', 0, 'SIGTERM') + + expect(exits).toEqual([true]) + }) + it('waits for the live tree refresh before ending stdin', async () => { const refreshDone = Promise.withResolvers() mocks.refresh.mockReturnValueOnce(refreshDone.promise) diff --git a/src/main/claude/claude-stream-json-connection.ts b/src/main/claude/claude-stream-json-connection.ts index 6f67e04b36ea..b7c50646fb46 100644 --- a/src/main/claude/claude-stream-json-connection.ts +++ b/src/main/claude/claude-stream-json-connection.ts @@ -65,7 +65,9 @@ export type ClaudeStreamJsonConnectionHandlers = { onUserDialog?: OnUserDialog /** A transport/process fault that is not itself first-hand root exit proof. */ onFault?: (error: Error) => void - onExit?: (error: Error) => void + /** The root process exited, reported once. `expected`: a close had begun, so it is that close's + * end, even one that ran out of its own escalation first and came back unproven. */ + onExit?: (error: Error, exit?: { expected: boolean }) => void } /** @@ -222,9 +224,9 @@ export async function openClaudeStreamJsonConnection( faultReported = true handlers.onFault?.(terminalError) } - if (!closing && exited && !exitReported) { + if (exited && !exitReported) { exitReported = true - handlers.onExit?.(terminalError) + handlers.onExit?.(terminalError, { expected: closing }) } } diff --git a/src/main/claude/claude-structured-requested-stop.test.ts b/src/main/claude/claude-structured-requested-stop.test.ts index 993109f4a378..edcaff847bb8 100644 --- a/src/main/claude/claude-structured-requested-stop.test.ts +++ b/src/main/claude/claude-structured-requested-stop.test.ts @@ -85,6 +85,7 @@ describe('a requested stop of a structured Claude chat', () => { expect(turns.at(-1)).toMatchObject({ turnId: 'turn-1', state: 'interrupted' }) const ended = events.filter((event) => event.type === 'ended') expect(ended).toEqual([expect.objectContaining({ reason: 'claude session closed' })]) - expect(ended[0]).not.toHaveProperty('cause') + // The host ends the child's record on it, as the close Orca asked for, never as a crash. + expect(ended[0]).toMatchObject({ cause: 'requested-close' }) }) }) diff --git a/src/main/claude/claude-structured-session-acquisition.ts b/src/main/claude/claude-structured-session-acquisition.ts index 404787620e95..39d2e0cdbe2f 100644 --- a/src/main/claude/claude-structured-session-acquisition.ts +++ b/src/main/claude/claude-structured-session-acquisition.ts @@ -153,12 +153,9 @@ export async function acquireClaudeSession({ settle() } } - const { canUseTool, onUserDialog } = buildClaudePermissionCallbacks({ - sessionId, - prompts, - emit: (event) => - callbacks.deliver(attempt, sessionId, () => callbacks.emit(liveSession, input.events, event)) - }) + const emit = (event: Parameters[2]): void => + callbacks.deliver(attempt, sessionId, () => callbacks.emit(liveSession, input.events, event)) + const { canUseTool, onUserDialog } = buildClaudePermissionCallbacks({ sessionId, prompts, emit }) try { const launch = await resolveClaudeAcquisitionLaunch({ @@ -200,7 +197,12 @@ export async function acquireClaudeSession({ childEnded ??= error initProof.reject(error) }, - onExit: (error) => { + onExit: (error, exit) => { + if (exit?.expected) { + // The end of a close Orca began; that close settles it, or finishes it now. + callbacks.finishClose(sessionId, attempt) + return + } // The child exited on its own; marked in place, as the fault report may hold this error. withObservedProviderExit(error) childEnded ??= error @@ -218,8 +220,6 @@ export async function acquireClaudeSession({ deps.now ? { now: deps.now } : {} ) acquisitions.assertCurrent(sessionId, attempt) - const emit = (event: Parameters[2]): void => - callbacks.deliver(attempt, sessionId, () => callbacks.emit(liveSession, input.events, event)) if (connection.pid === undefined) { // A pid-less spawn always reports its error next; surface that, not the missing pid. await initProof.promise diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index 746d603308f1..a752de5165d9 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -26,7 +26,11 @@ import { type ClaudeStructuredSessionAdapterDeps, type ClaudeStructuredSessionEvent } from './claude-structured-session-state' -import { closeAllClaudeSessions, closeClaudeSession } from './claude-structured-session-close' +import { + closeAllClaudeSessions, + closeClaudePublishedSessionForDeps, + closeClaudeSession +} from './claude-structured-session-close' import { claudeStoppedRequestEndWait } from './claude-request-end-wait' import { drainClaudeObservedExits, @@ -95,6 +99,14 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda emit: (session, _events, event) => this.emit(session, event), handleExit: (sessionId, attempt, error) => observeClaudeSessionExit(this.exitLifecycle, sessionId, attempt, error), + finishClose: (sessionId, attempt) => { + if (this.sessions.get(sessionId)?.connection === attempt.connection) { + // Joins a close still running; finalizes one that came back unproven before the exit. + void closeClaudePublishedSessionForDeps(this.sessions, sessionId, this.deps).catch( + () => undefined + ) + } + }, settleExit: (sessionId, exit) => settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit) } diff --git a/src/main/claude/claude-structured-session-close.test.ts b/src/main/claude/claude-structured-session-close.test.ts index 8881c6ae74b8..0e67b22ade4a 100644 --- a/src/main/claude/claude-structured-session-close.test.ts +++ b/src/main/claude/claude-structured-session-close.test.ts @@ -126,7 +126,11 @@ describe('Claude published session close lifecycle', () => { ).sessions.get('session-1') const disposeTranslator = vi.spyOn(session!.translator!, 'dispose') - await expect(adapter.closeSession('session-1')).rejects.toBe(persistenceError) + // The exit is proven; the failed cursor write is reported, never read as an unproven exit. + await expect(adapter.closeSession('session-1')).rejects.toMatchObject({ + name: 'AgentSessionAcquisitionExitProvenError', + cause: persistenceError + }) // The child is provably dead; a failed cursor write may not suppress the end. expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) expect(events.filter((event) => event.type === 'handle')).toHaveLength(0) @@ -134,10 +138,9 @@ describe('Claude published session close lifecycle', () => { // A close Orca asked for stops the child still running, then the host hears the session end. expect(childWork).toEqual(['live', 'ended', 'session-ended']) + // Nothing of the dead child stays indexed for a retry. await expect(adapter.closeSession('session-1')).resolves.toBe(true) - expect(persistHandle).toHaveBeenCalledTimes(2) - // The retry persists the same cursor without a second lifecycle end. - expect(events.filter((event) => event.type === 'handle')).toHaveLength(1) + expect(persistHandle).toHaveBeenCalledOnce() expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) expect(disposeTranslator).toHaveBeenCalledOnce() }) diff --git a/src/main/claude/claude-structured-session-close.ts b/src/main/claude/claude-structured-session-close.ts index 68f3dbbae1fe..05d767054094 100644 --- a/src/main/claude/claude-structured-session-close.ts +++ b/src/main/claude/claude-structured-session-close.ts @@ -7,6 +7,7 @@ import type { } from './claude-structured-session-state' import { cancelClaudeAcquisitionAttempt } from './claude-structured-session-state' import { + AgentSessionAcquisitionExitProvenError, AgentSessionAcquisitionExitUnprovenError, AgentSessionAcquisitionRootExitObservedError, AgentSessionPreSpawnError @@ -137,6 +138,10 @@ async function finalizeClaudePublishedSession( type: 'ended', sessionId: input.sessionId, reason: 'claude session closed', + // The host ends the child's record on it, whoever was still waiting on the close. + cause: 'requested-close', + fence: session.fence, + acquisitionGeneration: session.acquisitionGeneration, observedAt: Date.now() } as const let callbackError: unknown @@ -152,8 +157,6 @@ async function finalizeClaudePublishedSession( let persistenceError: unknown try { await persistence - session.closeFinalized = true - input.sessions.delete(input.sessionId) deliver({ type: 'handle', sessionId: input.sessionId, @@ -162,16 +165,13 @@ async function finalizeClaudePublishedSession( fence: session.fence }) } catch (error) { - // Keep the closed session indexed so a retry can persist the same cursor. - // Removing it first would turn a durable-write failure into a no-op retry. - if (session.closePersistence === persistence) { - session.closePersistence = undefined - } persistenceError = error } - // The connection already proved the child dead, so the session has ended - // whatever the durable write did: withholding it would strand the renderer on - // a session nothing re-drives. Emitted once, so a retry only re-persists. + // The connection already proved the child dead, so the session has ended whatever the resume + // point's write did: that write is bookkeeping, reported below, never a reason to keep a dead + // child indexed as if its exit were unproven. + session.closeFinalized = true + input.sessions.delete(input.sessionId) if (!session.closeEnded) { session.closeEnded = true try { @@ -186,15 +186,15 @@ async function finalizeClaudePublishedSession( session.translator?.dispose() } } - if (persistenceError) { - throw persistenceError - } - if (callbackThrew) { - throw callbackError - } if (rootExitVerdict) { throw rootExitVerdict } + if (persistenceError !== undefined || callbackThrew) { + // The exit is proven; only what followed it failed, which the caller reports. + throw new AgentSessionAcquisitionExitProvenError( + persistenceError !== undefined ? persistenceError : callbackError + ) + } return true } diff --git a/src/main/claude/claude-structured-session-recovery.test.ts b/src/main/claude/claude-structured-session-recovery.test.ts index 71a7ac9ce319..7005f93f5837 100644 --- a/src/main/claude/claude-structured-session-recovery.test.ts +++ b/src/main/claude/claude-structured-session-recovery.test.ts @@ -95,13 +95,16 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { ).sessions.get('session-1') const disposeTranslator = vi.spyOn(session!.translator!, 'dispose') - await expect(adapter.closeSession('session-1')).rejects.toBe(callbackError) + await expect(adapter.closeSession('session-1')).rejects.toMatchObject({ + name: 'AgentSessionAcquisitionExitProvenError', + cause: callbackError + }) expect(events.filter((event) => event.type === 'handle')).toHaveLength(1) expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) expect(disposeTranslator).toHaveBeenCalledOnce() }) - it('retains a closed session until its durable cursor persistence succeeds', async () => { + it('does not keep a dead child indexed over a failed resume-point write', async () => { const claude = fakeClaude() const persistenceError = new Error('store unavailable') const persistHandle = vi @@ -111,10 +114,13 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { const adapter = adapterFor(claude, {}, [], [], undefined, persistHandle) await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) - await expect(adapter.closeSession('session-1')).rejects.toBe(persistenceError) - expect(persistHandle).toHaveBeenCalledTimes(1) + // The exit is proven, so the close reports the write and still ends the session. + await expect(adapter.closeSession('session-1')).rejects.toMatchObject({ + name: 'AgentSessionAcquisitionExitProvenError', + cause: persistenceError + }) await expect(adapter.closeSession('session-1')).resolves.toBe(true) - expect(persistHandle).toHaveBeenCalledTimes(2) + expect(persistHandle).toHaveBeenCalledOnce() }) it('persists the last completed turn message before graceful close', async () => { diff --git a/src/main/claude/claude-structured-session-state.ts b/src/main/claude/claude-structured-session-state.ts index d61df04ac75c..670f28e5531b 100644 --- a/src/main/claude/claude-structured-session-state.ts +++ b/src/main/claude/claude-structured-session-state.ts @@ -345,5 +345,7 @@ export type ClaudeAcquireCallbacks = { event: ClaudeStructuredSessionEvent ) => void handleExit: (sessionId: string, attempt: ClaudeAcquisitionAttempt, error: Error) => void + /** The root exited during a close Orca began: finishes that close for this exact child. */ + finishClose: (sessionId: string, attempt: ClaudeAcquisitionAttempt) => void settleExit: (sessionId: string, exit: ClaudeSessionExit) => Promise } diff --git a/src/main/codex/codex-app-server-connection-types.ts b/src/main/codex/codex-app-server-connection-types.ts index df3f40ac39a5..8c6d4c13d9dd 100644 --- a/src/main/codex/codex-app-server-connection-types.ts +++ b/src/main/codex/codex-app-server-connection-types.ts @@ -8,7 +8,9 @@ export type CodexAppServerConnectionHandlers = { onNotification?: (method: string, params: unknown) => void onServerRequest?: (request: CodexAppServerServerRequest) => void onUnhandledFrame?: (kind: string, payload: unknown) => void - onExit?: (error: Error) => void + /** The root process exited, reported once. `expected`: a close had begun, so it is that close's + * end, even one that gave up first and came back unproven. */ + onExit?: (error: Error, exit?: { expected: boolean }) => void /** Awaited once the child has a pid and before the handshake; a rejection reaps the child. */ onSpawned?: (pid: number) => Promise } diff --git a/src/main/codex/codex-app-server-connection.test.ts b/src/main/codex/codex-app-server-connection.test.ts index 3d95e7ce380f..962270423e33 100644 --- a/src/main/codex/codex-app-server-connection.test.ts +++ b/src/main/codex/codex-app-server-connection.test.ts @@ -812,13 +812,13 @@ describe('openCodexAppServerConnection', () => { await connection.close() }) - it('keeps a graceful close quiet when stdin breaks during the reap', async () => { + it('reports a graceful close as expected when stdin breaks during the reap', async () => { const { child, spawnImpl } = stubChild({ exitOnStdinEnd: false }) answerInitialize(child) - const exits: string[] = [] + const exits: (boolean | undefined)[] = [] const connection = await openCodexAppServerConnection( { command: 'codex', args: ['app-server'] }, - { onExit: (error) => exits.push(error.message) }, + { onExit: (_error, exit) => exits.push(exit?.expected) }, spawnImpl ) child.stdin.on('finish', () => child.stdin.emit('error', new Error('write EPIPE'))) @@ -831,6 +831,7 @@ describe('openCodexAppServerConnection', () => { await connection.close() expect((await inFlight).message).toContain('EPIPE') - expect(exits).toHaveLength(0) + // The root's exit is the close's own end, never an unexpected death. + expect(exits).toEqual([true]) }) }) diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index 3e7d16ea1bb0..e68761040351 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -126,9 +126,9 @@ export async function openCodexAppServerConnection( // Transport/protocol failures make the connection unusable immediately so // callers do not hang, but recovery must not treat that as a child exit // until the execution host has observed `exit`/`close`. - if (exitObserved && !closing && !exitReported) { + if (exitObserved && !exitReported) { exitReported = true - handlers.onExit?.(terminalError) + handlers.onExit?.(terminalError, { expected: closing }) } } @@ -257,11 +257,17 @@ export async function openCodexAppServerConnection( ) if (!exited) { const treeExited = await terminateProcessTree() - if (!treeExited) { - dispatcher.failPending(new Error('codex app-server process-tree exit was not proven')) - return false - } await waitForProcessExitUntil(exitPromise, FORCED_EXIT_MS) + if (!treeExited && exitObserved) { + // The lease follows the root, which is gone: a child left behind is reported, and + // blocks nothing. + console.warn( + '[codex-app-server] root exited but its process tree was not proven gone', + { + pid: child.pid + } + ) + } } } dispatcher.failPending(new Error('codex app-server connection closed')) diff --git a/src/main/codex/codex-requested-close-turn-timing.test.ts b/src/main/codex/codex-requested-close-turn-timing.test.ts index a7baf6a14b59..45754f9f8042 100644 --- a/src/main/codex/codex-requested-close-turn-timing.test.ts +++ b/src/main/codex/codex-requested-close-turn-timing.test.ts @@ -1,7 +1,6 @@ import { createCodexDispatchEchoes } from './codex-structured-dispatch-echo' import { createCodexTurnOpenWaits } from './codex-structured-turn-open-wait' import { afterEach, describe, expect, it, vi } from 'vitest' -import type { AgentJournalItemBody } from '../../shared/agent-session-journal-types' import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' import { CodexBackgroundTaskTracker } from './codex-background-task-tracker' import { createCodexJournalTranslator } from './codex-structured-journal-translation' @@ -12,28 +11,19 @@ import type { CodexSession } from './codex-structured-session-state' afterEach(() => vi.useRealTimers()) describe('requested-close durable turn timing', () => { + // The exit is observed, so a refused terminal row is the host's to settle, never a reason to + // keep the dead child indexed for a retry. it.each([true, false])( - 'keeps the first exit receipt when retry requestedClose=%s', + 'ends the session with its exit receipt though its terminal row is refused (requestedClose=%s)', async (requestedClose) => { vi.useFakeTimers() vi.setSystemTime(1_000) - const terminalBodies: AgentJournalItemBody[] = [] - let refuseSettlement = true + // Backpressure refuses the terminal row the close's end would write. const sink: StructuredAgentSessionEventSink = { appendItem: () => {}, appendTombstone: () => {}, publish: () => {}, - tryAppendLifecycleBatch: (_id, mutations) => { - if (refuseSettlement) { - return { accepted: false, reason: 'backpressure' } - } - for (const mutation of mutations) { - if (mutation.kind === 'item') { - terminalBodies.push(mutation.body) - } - } - return { accepted: true } - } + tryAppendLifecycleBatch: () => ({ accepted: false, reason: 'backpressure' }) } const translator = createCodexJournalTranslator({ sink, @@ -80,22 +70,11 @@ describe('requested-close durable turn timing', () => { const onEvent = vi.fn() vi.setSystemTime(2_000) - await expect(closeCodexPublishedSession(sessions, 'session-1')).resolves.toBe(false) - expect(sessions.get('session-1')).toBe(session) - expect(session.ended).toBe(false) - - refuseSettlement = false - vi.setSystemTime(60_000) - await expect( - closeCodexPublishedSession(sessions, 'session-1', onEvent, { - expectedAcquisitionGeneration: 'replacement-generation' - }) - ).resolves.toBe(false) - expect(onEvent).not.toHaveBeenCalled() await expect( closeCodexPublishedSession(sessions, 'session-1', onEvent, { requestedClose }) ).resolves.toBe(true) expect(sessions.has('session-1')).toBe(false) + expect(session.ended).toBe(true) expect(onEvent).toHaveBeenCalledWith( expect.objectContaining({ cause: requestedClose ? 'requested-close' : 'unexpected-exit', @@ -103,12 +82,6 @@ describe('requested-close durable turn timing', () => { observedAt: 2_000 }) ) - expect(terminalBodies.find((body) => body.kind === 'turn')).toMatchObject({ - kind: 'turn', - state: 'interrupted', - startedAt: 1_000, - completedAt: 2_000 - }) } ) }) diff --git a/src/main/codex/codex-structured-session-acquire.ts b/src/main/codex/codex-structured-session-acquire.ts index 37c34a4bf3a1..bf40afa199bd 100644 --- a/src/main/codex/codex-structured-session-acquire.ts +++ b/src/main/codex/codex-structured-session-acquire.ts @@ -163,13 +163,15 @@ export async function acquireCodexStructuredSession(input: { Buffer.byteLength(JSON.stringify(payload ?? null), 'utf8') ), onSpawned: spawnIdentity.onSpawned, - onExit: (error) => { + onExit: (error, exit) => { try { handleCodexSessionExit({ sessions, sessionId, connection: acquisition.connection, error, + // The end of a close Orca began, even one that came back unproven before it. + ...(exit?.expected ? { closedByOrca: true as const } : {}), prompts: acquisition.prompts, ...(deps.onEvent ? { onEvent: deps.onEvent } : {}) }) diff --git a/src/main/codex/codex-structured-session-close.test.ts b/src/main/codex/codex-structured-session-close.test.ts index 9d80a0fa9829..73004e43c344 100644 --- a/src/main/codex/codex-structured-session-close.test.ts +++ b/src/main/codex/codex-structured-session-close.test.ts @@ -135,7 +135,7 @@ describe('Codex structured session close lifecycle', () => { expect(translator.handle).toHaveBeenCalledOnce() }) - it("ends a Stop's wait for its turn to open when a requested close cannot publish its end yet", async () => { + it("ends the session and a Stop's wait for its turn to open when a requested close cannot publish its end", async () => { const { connection, prompts, session, sessions } = backpressuredSession(true) let released = false void session.turnOpenWaits.wait('turn-1', 60_000).then(() => { @@ -151,10 +151,10 @@ describe('Codex structured session close lifecycle', () => { closedByOrca: true, prompts }) - ).toBe(false) + ).toBe(true) await Promise.resolve() - // Left for the retry, but the child is gone: nothing waits on a turn it would open. - expect(session.ended).toBe(false) + // The exit is observed: the refused row is the host's to settle, and nothing waits on a turn. + expect(session.ended).toBe(true) expect(released).toBe(true) }) diff --git a/src/main/codex/codex-structured-session-close.ts b/src/main/codex/codex-structured-session-close.ts index fe6c0d676735..0afd70a70e8c 100644 --- a/src/main/codex/codex-structured-session-close.ts +++ b/src/main/codex/codex-structured-session-close.ts @@ -14,11 +14,9 @@ export function handleCodexSessionExit(input: { sessionId: string connection: CodexAppServerConnection | null error: Error - /** Set by Orca's own close. Absent only from the connection's onExit, which the connection - * withholds while Orca is closing the child. */ + /** Set for the end of a close Orca began: by that close, or by the root's own exit report. */ closedByOrca?: true prompts?: CodexSession['prompts'] - allowFailedSettlement?: boolean onEvent?: (event: CodexStructuredSessionEvent) => void }): boolean { const session = input.sessions.get(input.sessionId) @@ -43,13 +41,9 @@ export function handleCodexSessionExit(input: { observedAt: session.exitObservedAt } as const // A synchronous sink rejection (usually backpressure) leaves the terminal rows to the host's - // exit settlement, which writes its own bounded fallback. - const admission = session.translator?.handle(event) ?? { accepted: true } - // The connection invokes onExit exactly once, so an unexpected exit is forwarded even when - // admission is backpressured; waiting for a second callback would strand the lease. - if (!admission.accepted && event.cause !== 'unexpected-exit' && !input.allowFailedSettlement) { - return false - } + // exit settlement, which writes its own bounded fallback. The exit itself is observed, so the + // session ends either way: holding a dead child as unproven over a refused row would strand it. + session.translator?.handle(event) session.ended = true // Nothing can echo for this child any more; the journal's pending-submission // recovery is what settles the sends these were armed for. @@ -69,7 +63,6 @@ export async function closeCodexPublishedSession( sessionId: string, onEvent?: (event: CodexStructuredSessionEvent) => void, options?: { - allowFailedSettlement?: boolean requestedClose?: boolean expectedFence?: number expectedAcquisitionGeneration?: string @@ -97,21 +90,15 @@ export async function closeCodexPublishedSession( return false } if (!session.ended) { - const handled = handleCodexSessionExit({ + handleCodexSessionExit({ sessions, sessionId, connection: session.connection, error: options?.unexpectedReason ?? new Error('codex session closed'), closedByOrca: true, prompts: session.prompts, - ...(options?.allowFailedSettlement ? { allowFailedSettlement: true } : {}), ...(onEvent ? { onEvent } : {}) }) - // Keep the closed session indexed when terminal settlement admission was - // rejected; a later close attempt retries the same stable lifecycle event. - if (!handled) { - return false - } } sessions.delete(sessionId) return true diff --git a/src/main/codex/codex-structured-session-teardown.ts b/src/main/codex/codex-structured-session-teardown.ts index 80b06ab1c36a..115e794d8310 100644 --- a/src/main/codex/codex-structured-session-teardown.ts +++ b/src/main/codex/codex-structured-session-teardown.ts @@ -40,7 +40,7 @@ export class CodexStructuredSessionTeardown { this.deps.sessions, sessionId, this.deps.onEvent, - { allowFailedSettlement: true, requestedClose: false } + { requestedClose: false } ) return this.settled(sessionId, closed) } @@ -63,7 +63,6 @@ export class CodexStructuredSessionTeardown { return Promise.resolve(false) } return closeCodexPublishedSession(this.deps.sessions, sessionId, this.deps.onEvent, { - allowFailedSettlement: true, requestedClose: false, expectedFence: fence, expectedAcquisitionGeneration: acquisitionGeneration, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-exit-proof.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-exit-proof.ts index 9d27e570db4a..1545a8fa618d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-exit-proof.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-exit-proof.ts @@ -50,16 +50,24 @@ function provenExitAcquisitionFailure(cause: unknown): unknown { } /** Whether a stop left the provider root gone. The lease follows the root, so a first-hand root - * exit or a processless child ends the session whatever its descendants did; any other - * failure still throws. */ -export async function stopAgentSessionProviderRoot(stop: () => Promise): Promise { + * exit or a processless child ends the session whatever its descendants did. A descendant left + * unconfirmed, or bookkeeping that failed after the proven exit, is only `report`ed. Any other + * failure throws. */ +export async function stopAgentSessionProviderRoot( + stop: () => Promise, + report?: (error: Error) => void +): Promise { try { return (await stop()) === true } catch (error) { if ( error instanceof AgentSessionAcquisitionRootExitObservedError || - isAgentSessionPreSpawnError(error) + error instanceof AgentSessionAcquisitionExitProvenError ) { + report?.(error) + return true + } + if (isAgentSessionPreSpawnError(error)) { return true } throw error diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index 3785fdeaf5ce..4c621a5e0c3b 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -271,7 +271,8 @@ async function installOnJournal( onDispatchSettledLate, onPrimaryThreadStoppedRunning: releaseUnansweredDispatches, onEvent: (event) => { - if (event.type === 'ended' && 'cause' in event && event.cause === 'unexpected-exit') { + // Every exit, expected or not: the host ends that child's record. + if (event.type === 'ended' && 'cause' in event) { lifecycle.deliver(event) } } diff --git a/src/main/runtime/structured-claude-runtime-adapter.ts b/src/main/runtime/structured-claude-runtime-adapter.ts index 9024c65b503d..246f5b5eea3f 100644 --- a/src/main/runtime/structured-claude-runtime-adapter.ts +++ b/src/main/runtime/structured-claude-runtime-adapter.ts @@ -47,9 +47,10 @@ export function structuredClaudeLifecycleEvent( if (event.type === 'started') { return event } + // Every exit of a child with an identity, expected or not: the host ends that child's record. if ( event.type === 'ended' && - event.cause === 'unexpected-exit' && + event.cause !== undefined && event.fence !== undefined && event.acquisitionGeneration ) { From b9667b52ea6e5b490b59c967aa03d88d17e846c5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:31:19 -0700 Subject: [PATCH 02/22] fix(native-chat): the exit ends the child's record; an unfinished stop is the child's own close, which everyone joins - The host keeps no stored "stop still owed" record any more. A stop begins the child's close (`child.close`), which lives on the child and ends with it. A second Stop, the idle reaper, quit, a send and an option/answer/goal/rewind all join that close instead of retrying a separate obligation. - A caller waits on the close only as long as the step deadline; the close itself is never abandoned. A proof that lands after every caller stopped waiting reaches the host as the adapter's report of that exit, which ends the record through the same handler. - Once the exit is proven, draining, settling, the lease release and the adapter's acknowledgement are each attempted and reported on failure; none keeps the child on record. A start, and the handle's close, write a release that failed from this host's proof of that exit, so a failed write never refuses a send. - A start that meets a close still unverifiable is refused with `previousExitUnverifiable`, so the queued message is rejected with a send-again reason; nothing is held and nothing starts beside the old process. - The idle sweep goes back to idle reaping only. - Removes #24333's retry entry points, the wait row and its hold rule, the ask/failure cursors on the stored record, and the stop's own wake. Tests replace the #24333 unproven-stop test: a send joining an unproven close and an in-flight one, a late proof past the caller's bound, a root exiting after its close gave up, a proven exit whose resume-point write and lease release both failed, an unverifiable close rejecting the send and refusing an option change, a surviving descendant, quit and the idle reaper; and Codex's unverifiable, late-exit and joined-close cases. --- ...-agent-session-accept-then-deliver.test.ts | 10 +- .../structured-agent-session-agent-start.ts | 60 +- .../structured-agent-session-chat-stop.ts | 15 + .../structured-agent-session-child-close.ts | 263 ++++++++ ...red-agent-session-claude-root-exit.test.ts | 3 +- ...ssion-claude-stop-exit-ends-record.test.ts | 470 +++++++++++++++ ...-session-claude-unproven-stop-send.test.ts | 569 ------------------ ...ctured-agent-session-close-verdict.test.ts | 15 +- ...tured-agent-session-codex-stop-row.test.ts | 79 ++- ...d-agent-session-conversation-close.test.ts | 65 +- ...red-agent-session-conversation-lifetime.ts | 14 +- ...ed-agent-session-conversation-stop.test.ts | 3 +- .../structured-agent-session-delivery-loop.ts | 22 +- ...structured-agent-session-event-recovery.ts | 18 +- ...uctured-agent-session-eviction-deadline.ts | 13 +- .../structured-agent-session-eviction.test.ts | 154 +---- .../structured-agent-session-eviction.ts | 113 ++-- .../structured-agent-session-host-lifetime.ts | 232 ++----- ...tructured-agent-session-host-stop-event.ts | 21 +- .../structured-agent-session-host-types.ts | 23 +- .../structured-agent-session-host.ts | 13 +- ...tructured-agent-session-idle-sweep.test.ts | 1 - .../structured-agent-session-idle-sweep.ts | 9 - ...ructured-agent-session-mutation-context.ts | 6 +- ...structured-agent-session-provider-child.ts | 18 +- ...ructured-agent-session-send-preparation.ts | 7 +- ...d-agent-session-stop-event-entries.test.ts | 13 +- ...red-agent-session-surface-lifetime.test.ts | 30 +- .../structured-agent-session-turns-cancel.ts | 6 +- ...tructured-agent-session-unexpected-exit.ts | 3 + ...ctured-agent-session-wind-down-wait-row.ts | 83 --- 31 files changed, 1032 insertions(+), 1319 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-stop-send.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-wind-down-wait-row.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts index b3f91aea7002..1b4a4e95b8c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -773,9 +773,10 @@ describe('an eviction between acceptance and handover', () => { }) }) -// A close abandons what is queued before it stops the child, so a release that then fails still -// leaves every queued message rejected as closed, never blamed on the provider. -describe('a close that stops the child and then fails', () => { +// A close abandons what is queued before it stops the child, so a wind-down step that then fails +// (reported, never the close's failure) still leaves every queued message rejected as closed, +// never blamed on the provider. +describe('a close that stops the child and then a wind-down step fails', () => { const END_CHILD = { evict: () => host.close(SESSION, 'evict') } satisfies Partial Promise>> @@ -803,8 +804,7 @@ describe('a close that stops the child and then fails', () => { const id = await accept('hello') await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) - await expect(END_CHILD[end]()).rejects.toThrow() - expect(host.hasSession(SESSION)).toBe(true) + await expect(END_CHILD[end]()).resolves.toBeUndefined() started.resolve() await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts index d2ef237aa1b6..ec1289d401dd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts @@ -29,9 +29,10 @@ import { attachStructuredAgentSessionUnderSerialize } from './structured-agent-s import { failedCreateRefusal } from './structured-agent-session-failed-create-refusal' import { adapterSupportsRecord } from './structured-agent-session-provider-support' import { - finishOwedStructuredAgentSessionWindDownUnderSerialize, - type StructuredAgentSessionLifetimeContext -} from './structured-agent-session-host-lifetime' + joinClosingStructuredAgentSessionChild, + releaseLeaseOfEndedStructuredAgentSessionChild +} from './structured-agent-session-child-close' +export { joinClosingStructuredAgentSessionChild } import { structuredAgentSessionResumeOperationId, structuredAgentSessionResumeParams @@ -52,48 +53,6 @@ export type StructuredAgentSessionResumeOutcome = /** The attach's caller key: the ledger row a start settles is Orca's own. */ const AGENT_START_CALLER_KEY = 'trusted-local:agent-start' -/** - * Every operation that reaches the provider finishes a stop an earlier attempt left owed first: the - * child that stop could not prove gone takes no input, and none may start beside it. Still - * unproven, the operation is refused with the exit `unverifiable`, never assumed `exited`. - */ -export async function finishOwedStructuredAgentSessionStop( - context: StructuredAgentSessionLifetimeContext, - sessionId: string -): Promise { - if (await finishOwedStructuredAgentSessionWindDownUnderSerialize(context, sessionId)) { - return { ok: true } - } - return { - ok: false, - refusal: refuse( - 'agent_session_ownership_unknown', - { reason: 'previousExitUnverifiable', ownerVerdict: 'unverifiable' }, - "Orca could not prove this chat's previous agent process exited." - ) - } -} - -/** The same, for an operation the running child performs, which starts none: only an exit still - * unproven, its child still on record, holds it back. A proven exit owes only bookkeeping, which - * gates no such operation: its retry was reported, and the operation goes on as with none owed. */ -export async function finishOwedStructuredAgentSessionStopForProviderWrite( - context: StructuredAgentSessionLifetimeContext, - sessionId: string -): Promise { - const settled = await finishOwedStructuredAgentSessionStop(context, sessionId) - return settled.ok || context.sessions.get(sessionId)?.child ? settled : { ok: true } -} - -export function isStructuredAgentSessionPreviousExitUnverifiable( - refusal: AgentSessionWireRefusal -): boolean { - return ( - refusal.code === 'agent_session_ownership_unknown' && - refusal.details?.reason === 'previousExitUnverifiable' - ) -} - /** * Gives the session a provider child if it has none, for a caller inside its serialize — which is * what makes "if it has none" exact: two askers run this in turn, and the second finds the first @@ -104,9 +63,11 @@ export async function ensureStructuredAgentSessionAgent( sessionId: string, startedFor?: string ): Promise { - const settled = await finishOwedStructuredAgentSessionStop(context, sessionId) - if (!settled.ok) { - return settled + // A child a stop began closing takes no input, and none may start beside it: the start waits on + // that close, and is refused while its exit stays unverifiable. + const closed = await joinClosingStructuredAgentSessionChild(context, sessionId) + if (!closed.ok) { + return closed } if (context.sessions.get(sessionId)?.child) { return { ok: true } @@ -158,6 +119,9 @@ async function startStructuredAgentSessionAgent( return { ok: false, refusal: unreconciled } } await context.runtimeState.resolveRecovery(sessionId) + // A start needs the lease released; a release the exit's own wind-down could not write is + // re-derived from this host's proof of that exit, never refused on. + await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId) const record = context.deps.store.getRecord(sessionId) if (!record) { return refuseResume( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts index 77a6fa3434e0..75d2a44e1b89 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts @@ -73,6 +73,21 @@ export function mutateWithChatStop( agentSessionFailureWords(agentSessionFailureFact('cancelled'), { surface: 'rejection' }) ) const child = context.sessions.get(ctx.sessionId)?.child + if (child?.close) { + // A close an earlier stop began: this Stop joins it, retrying the exit's proof, rather + // than asking a child that takes no input to stop again. + if (withdrawn.length > 0) { + await tookEffect() + } + await stopChild().catch((error: unknown) => + context.deps.logger.warn('ending the agent process on Stop failed', { + scope: 'stop-child', + sessionId, + error + }) + ) + return { ok: true, value: { ...named, cancelled: withdrawn.length > 0 } } + } if (child?.phase === 'starting') { // A start that may never land is the one thing here Stop has to end; the chat stays. await tookEffect() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts new file mode 100644 index 000000000000..27bcd9c6fb53 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -0,0 +1,263 @@ +// A provider child's close: one per child, joined by every stop, start and provider write that +// meets it, and the one place a close's proven exit ends the child's record. +// +// The close lives on the child (`child.close`) and ends with it, so nothing outlives the process +// it is about. A caller waits a bounded time for the exit's proof; past the bound it answers +// `unverifiable` and the close keeps running. Whenever the exit is proven — by a caller still +// waiting, or by the adapter's own report of it — the record ends here. + +import { withTimeout } from '../../../shared/promise-timeout-fallback' +import { refuse } from '../../../shared/agent-session-wire-refusals' +import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { + evictStructuredAgentSession, + STRUCTURED_AGENT_SESSION_EVICTION_STEPS +} from './structured-agent-session-eviction' +import { + STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS, + withStructuredAgentSessionEvictionDeadline +} from './structured-agent-session-eviction-deadline' +import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' +import type { + StructuredAgentSessionChildClose, + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild +} from './structured-agent-session-host-types' +import { endProviderChild } from './structured-agent-session-provider-child' +import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' +import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' +import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' +import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' +import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' + +/** What a caller learned about the child's exit: proven, or not within its bound. Loss of contact + * and a close that ran out of its own escalation both read `unverifiable`, never `exited`. */ +export type StructuredAgentSessionChildCloseVerdict = 'exited' | 'unverifiable' + +/** Joins the child's close, starting another attempt when the last one ended unproven, and waits + * for its proof as long as the caller may. A proof ends the record before this resolves. */ +export async function joinStructuredAgentSessionChildClose( + context: StructuredAgentSessionLifetimeContext, + sessionId: string, + child: StructuredAgentSessionProviderChild, + close: StructuredAgentSessionChildClose +): Promise { + close.attempt ??= closeAttempt(context, sessionId, close) + const proven = await withTimeout( + close.attempt, + STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS, + false + ) + if (!proven) { + return 'unverifiable' + } + await endClosedStructuredAgentSessionChildUnderSerialize(context, sessionId, child) + return 'exited' +} + +/** The refusal of an operation that met a child whose close is still unproven. */ +export function previousExitUnverifiableRefusal(): AgentSessionWireRefusal { + return refuse( + 'agent_session_ownership_unknown', + { reason: 'previousExitUnverifiable', ownerVerdict: 'unverifiable' }, + "Orca could not prove this chat's previous agent process exited." + ) +} + +/** For an operation that reaches the provider: a child a stop began closing takes no input and + * none may start beside it, so the operation joins that close and is refused while it is still + * unproven. */ +export async function joinClosingStructuredAgentSessionChild( + context: StructuredAgentSessionLifetimeContext, + sessionId: string +): Promise<{ ok: true } | { ok: false; refusal: AgentSessionWireRefusal }> { + const child = context.sessions.get(sessionId)?.child + if (!child?.close) { + return { ok: true } + } + return (await joinStructuredAgentSessionChildClose(context, sessionId, child, child.close)) === + 'exited' + ? { ok: true } + : { ok: false, refusal: previousExitUnverifiableRefusal() } +} + +function closeAttempt( + context: StructuredAgentSessionLifetimeContext, + sessionId: string, + close: StructuredAgentSessionChildClose +): Promise { + const { adapter, logger } = context.deps + // An adapter with no close has nothing to stop; anything else must PROVE the exit. + const stop = adapter.disposeSession ?? adapter.closeSession + const attempt = ( + stop + ? stopAgentSessionProviderRoot( + () => stop.call(adapter, sessionId), + // The exit is proven; a child process left behind, or a write after it, is reported only. + (error) => + logger.warn("the agent's process exited, but its close did not finish cleanly", { + scope: 'provider-close-after-exit', + sessionId, + error + }) + ) + : Promise.resolve(true) + ).catch((error: unknown) => { + logger.warn("closing the agent's process did not prove it exited", { + scope: 'provider-close', + sessionId, + error + }) + return false + }) + // Unproven, the next ask starts another attempt. A proof that lands after every caller stopped + // waiting needs nothing here: the adapter reports that exit, which ends the record. + void attempt.then((proven) => { + if (!proven && close.attempt === attempt) { + close.attempt = undefined + } + }) + return attempt +} + +/** Whether the lease still names the child this host last proved gone, unreleased: only that + * in-memory proof lets this host release it without a probe, so the handle carrying it stays. */ +export function structuredAgentSessionEndedChildHoldsLease( + context: Pick, + sessionId: string +): boolean { + const ended = context.sessions.get(sessionId)?.lastEndedChild + const record = context.deps.store.getRecord(sessionId) + return ( + ended?.rootGone === true && + record !== null && + isSurfaceReleasableAgentSessionRecord(record) && + record.lease.runtimeFence === ended.fence + ) +} + +/** Writes the release a proven exit allows when its wind-down could not: a start and the + * handle's close re-derive it, and a failure is reported, never anyone's refusal. Resolves + * whether the lease still names that child. */ +export async function releaseLeaseOfEndedStructuredAgentSessionChild( + context: Pick, + sessionId: string +): Promise { + const ended = context.sessions.get(sessionId)?.lastEndedChild + if (!ended || !structuredAgentSessionEndedChildHoldsLease(context, sessionId)) { + return false + } + await releaseStoredStructuredAgentSessionOwner({ + store: context.deps.store, + sessionId, + hasProviderChild: true, + expectedFence: ended.fence, + now: context.now() + }).catch((error: unknown) => + context.deps.logger.warn("releasing an exited agent's lease failed", { + scope: 'ended-child-lease-release', + sessionId, + error + }) + ) + return structuredAgentSessionEndedChildHoldsLease(context, sessionId) +} + +/** The adapter's report that a child it was asked to close is gone ends the record: a proof that + * landed with no caller left waiting, or a root that exited after its close gave up. */ +export function endClosedStructuredAgentSessionChild( + context: StructuredAgentSessionLifetimeContext, + event: StructuredAgentSessionEndedEvent +): Promise { + return context.serialize(event.sessionId, async () => { + const child = context.sessions.get(event.sessionId)?.child + if (child && child.fence === event.fence && child.generation === event.acquisitionGeneration) { + await endClosedStructuredAgentSessionChildUnderSerialize(context, event.sessionId, child) + } + }) +} + +/** One wind-down per child, which every caller that saw its exit proven awaits. */ +const windDowns = new WeakMap>() + +/** + * The child's process is proven gone: it leaves the record, then every wind-down step is + * attempted, each failure reported, none able to keep it on record or gate the next send. Then + * the delivery loop hands over whatever is queued. Joins the wind-down already running for it. + */ +export function endClosedStructuredAgentSessionChildUnderSerialize( + context: StructuredAgentSessionLifetimeContext, + sessionId: string, + child: StructuredAgentSessionProviderChild +): Promise { + const running = windDowns.get(child) + if (running) { + return running + } + const session = context.sessions.get(sessionId) + if (!session || session.child !== child) { + return Promise.resolve() + } + const windDown = windDownClosedChild(context, sessionId, session, child) + windDowns.set(child, windDown) + return windDown +} + +async function windDownClosedChild( + context: StructuredAgentSessionLifetimeContext, + sessionId: string, + session: StructuredAgentSessionHostSession, + child: StructuredAgentSessionProviderChild +): Promise { + const { deps } = context + const close = child.close + endProviderChild(session, { + generation: child.generation, + fence: child.fence, + cause: close?.cause ?? 'evict', + reason: close?.reason ?? null, + duringStartup: child.phase === 'starting', + rootGone: true + }) + context.restartWitness?.stopped(sessionId) + await evictStructuredAgentSession( + { + sessionId, + eventSink: context.runtimeState.eventSinkFor(sessionId), + logger: deps.logger, + acknowledgeRelease: () => deps.adapter.acknowledgeSessionRelease?.(sessionId), + discardSink: () => context.runtimeState.discardEventSink(sessionId), + settleWork: async () => { + // Folded before the fallback's end is built, so the end reads it (`turnEndAfterStop`). + await close?.recorded + const settled = await settleStructuredAgentSessionDeadGeneration({ + journal: session.journal, + sessionId, + fence: child.fence, + settlementId: `expected-close:${sessionId}:${child.fence}:${child.generation ?? 'unknown'}`, + pendingSubmissionReason: 'provider_closed_before_acknowledgement', + // Only a turn no adapter settled: one with no close, or whose settle threw. Whether it + // was a person's Stop is its event's to say (`turnEndAfterStop`). + verdict: { state: 'interrupted', completedAt: context.now() }, + showUnexpectedExitOutcome: false + }) + if (!settled.ok) { + throw new Error('dead generation work settlement failed', { cause: settled.error }) + } + }, + releaseLease: () => + releaseStoredStructuredAgentSessionOwner({ + store: deps.store, + sessionId, + hasProviderChild: true, + expectedFence: child.fence, + now: context.now() + }) + }, + withStructuredAgentSessionEvictionDeadline(STRUCTURED_AGENT_SESSION_EVICTION_STEPS) + ) + // Whatever ended the child, the row belongs to the conversation: it shows not-running, and only + // the conversation's close forgets it. + context.publishStatus?.(sessionId) + context.wakeDelivery?.(sessionId) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index f09fe672b5ec..1867e4f4729f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -135,7 +135,8 @@ describe('Claude root-exit stop', () => { runtimeState, sessions, now: () => NOW + 30 * 60_000, - publishStatus + publishStatus, + serialize: (_sessionId, task) => task() }, 'session-1', { cause: 'evict' } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts new file mode 100644 index 000000000000..4e2ca9cf4b15 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -0,0 +1,470 @@ +// A Claude child's close and its exit, on the shipping adapter and host. A Stop begins the child's +// close; every later stop, start and provider write joins it; the exit, whenever it is proven, ends +// the child's record; and a close still unverifiable fails what needed the child, holding nothing. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-live-turn' +import { claudeUnwrittenUserMessageError } from '../../claude/claude-agent-sdk-user-message-queue' +import { ClaudeStructuredSessionAdapter } from '../../claude/claude-structured-session-adapter' +import { + fakeClaude, + PROVIDER_SESSION_ID, + type FakeConnection +} from '../../claude/claude-structured-session-test-support' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS } from './structured-agent-session-eviction-deadline' +import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +const CAPABILITIES = ['interrupt_receipt_v1', 'interrupt_cancel_queued_v1', 'msg_lifecycle_v1'] + +let root: string +let host: StructuredAgentSessionHost +let adapter: ClaudeStructuredSessionAdapter +let store: AgentSessionRecordStore +let claude: ReturnType +let log: ReturnType +let persistHandle: ReturnType Promise>> + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-claude-stop-exit-ends-record-')) + resetHostTestOperationIds() + log = recordingStructuredAgentSessionLogger() + persistHandle = vi.fn(async () => undefined) + claude = fakeClaude({ + replayUuid: null, + routes: { interrupt: () => ({ still_queued: [], cancelled: [] }) } + }) + const lifecycle: Promise[] = [] + adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: root, + claudeConfigDir: join(root, 'claude-home'), + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + resumesTranscript: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0, + continuesChain: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0 + }), + onEvent: (event) => { + const mapped = structuredClaudeLifecycleEvent(event) + if (mapped) { + lifecycle.push(host.handleAdapterEvent(mapped)) + } + }, + onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), + persistHandle: () => persistHandle(), + openConnection: claude.openConnection, + readProcessStartTime: async () => 1_700_000_000_000, + now: () => NOW + }) + store = await openTestAgentSessionRecordStore(root) + host = new StructuredAgentSessionHost({ + store, + adapter: Object.assign(adapter, { supportsCreate: () => true }), + journalDatabase: openTestJournalHostDatabase(root), + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + logger: log.logger, + // Ticked by hand only, and with no idle window: a tick reaps whatever is at rest. + idleSweep: { intervalMs: 3_600_000, idleMs: 0 }, + now: () => NOW + }) + const params = hostTestAttachParams(null, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION_ID, leafUuid: null } + }) + expect(await host.attach(CALLER, params)).toMatchObject({ ok: true }) + await adapter.awaitStarted(SESSION) + await Promise.all(lifecycle) +}) + +afterEach(async () => { + await adapter.closeAll() + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function eventually(assertion: () => T | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +function envelope( + method: + | 'agentSession.send' + | 'agentSession.cancel' + | 'agentSession.setOption' + | 'agentSession.queuedMessageSend', + // The fingerprint's own field shape, as the sibling host tests type it. + fields: Parameters[0]['fields'] +) { + return { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: store.getRecord(SESSION)!.lease.runtimeFence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method, + sessionId: SESSION, + fields + }) + } +} + +async function send(text: string): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + if (!sent.ok) { + throw new Error(`send refused: ${JSON.stringify(sent.refusal)}`) + } + return sent.value.clientMessageId +} + +async function submission(clientMessageId: string) { + await host.flushStreamedEvents(SESSION) + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) +} + +function frame(connection: FakeConnection, message: Record): void { + connection.handlers.onMessage?.({ session_id: PROVIDER_SESSION_ID, ...message }) +} + +function wrote(connection: FakeConnection, text: string): boolean { + return connection.sent.some((message) => JSON.stringify(message).includes(text)) +} + +async function openTurn(connection: FakeConnection): Promise { + const text = 'Write a long reply.' + const clientMessageId = await send(text) + await eventually(() => expect(wrote(connection, text)).toBe(true)) + frame(connection, { + type: 'system', + subtype: 'init', + uuid: 'init-1', + model: 'claude-sonnet-5', + capabilities: CAPABILITIES + }) + const written = connection.sent.at(-1)! + frame(connection, { ...written, uuid: written.uuid }) + frame(connection, { + type: 'assistant', + uuid: 'stopped-turn-leaf', + parent_tool_use_id: null, + message: { id: 'msg-1', role: 'assistant', content: [{ type: 'text', text: 'Working on' }] } + }) + await eventually(async () => + expect((await submission(clientMessageId))?.dispatchState).toBe('accepted') + ) + expect( + activeStructuredAgentSessionTurnId((await host.journalSnapshot(SESSION)).items) + ).not.toBeNull() +} + +function stop() { + return host.cancel(CALLER, { envelope: envelope('agentSession.cancel', {}) }) +} + +function laneDrained(): Promise { + return host['tasks'].serialize(SESSION, async () => {}) +} + +/** As the real connection: once a close begins it refuses every write, proven or not. */ +function closeUnprovenFor(connection: FakeConnection, failures: number): void { + const close = connection.close + let left = failures + connection.close = async () => { + if (left === 0) { + return close() + } + left -= 1 + connection.closeCount += 1 + connection.closed = true + return false + } + refuseWritesOnceClosed(connection) +} + +function refuseWritesOnceClosed(connection: FakeConnection): void { + const write = connection.send + connection.send = (message, beforeDispatch) => + connection.closed + ? Promise.reject( + claudeUnwrittenUserMessageError(new Error('claude stream-json connection is closed')) + ) + : write(message, beforeDispatch) +} + +const INTERRUPTED_RESULT = { + type: 'result', + subtype: 'error_during_execution', + is_error: true, + terminal_reason: 'aborted_streaming', + uuid: 'interrupted-result' +} + +function child() { + return host['sessions'].get(SESSION)?.child ?? null +} + +function lease() { + return store.getRecord(SESSION)?.lease +} + +/** Stops a running turn; its child's close cannot prove the exit `failures` times. */ +async function stopWithUnprovenClose(failures: number): Promise { + const connection = claude.connections[0]! + await openTurn(connection) + closeUnprovenFor(connection, failures) + await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) + frame(connection, INTERRUPTED_RESULT) + await laneDrained() + expect(connection.closeCount).toBe(1) + // The child stays on record with its close begun, never as a stored obligation beside it. + expect(child()?.close).toMatchObject({ cause: 'user-stop' }) + return connection +} + +async function resumedWith(connection: FakeConnection, text: string): Promise { + return eventually(() => { + const started = claude.connections.at(-1)! + expect(started).not.toBe(connection) + expect(wrote(started, text)).toBe(true) + return started + }) +} + +function setModel(model: string) { + const fields = { key: 'model', value: model } + return host.setOption(CALLER, { envelope: envelope('agentSession.setOption', fields), ...fields }) +} + +function deferred() { + let resolve: (value: T) => void = () => undefined + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +function scopes(): unknown[] { + return log.entries.map((entry) => entry.fields.scope) +} + +it('joins the close a Stop could not prove before the next message, then sends it to a resumed child', async () => { + const connection = await stopWithUnprovenClose(1) + + await send('Carry on.') + await resumedWith(connection, 'Carry on.') + + expect(connection.closeCount).toBe(2) + expect(wrote(connection, 'Carry on.')).toBe(false) + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ + cause: 'user-stop', + rootGone: true + }) +}) + +it('has a send made while the close still runs join that close, not start another', async () => { + const connection = claude.connections[0]! + await openTurn(connection) + const proof = deferred() + const close = connection.close + let attempts = 0 + connection.close = async () => { + attempts += 1 + connection.closed = true + await proof.promise + return close() + } + refuseWritesOnceClosed(connection) + const closeSession = vi.spyOn(adapter, 'closeSession') + await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) + frame(connection, INTERRUPTED_RESULT) + + await send('Carry on.') + await eventually(() => expect(attempts).toBe(1)) + expect(claude.connections).toHaveLength(1) + proof.resolve() + + await resumedWith(connection, 'Carry on.') + // One close of the old child, which the send waited on rather than asking for its own. + expect(closeSession).toHaveBeenCalledOnce() + expect(attempts).toBe(1) + expect(connection.closeCount).toBe(1) +}) + +// The caller's bound is how long it waits, never how long the close may take. +it('ends the record when the proof lands after every caller stopped waiting', async () => { + const connection = claude.connections[0]! + const proof = deferred() + connection.close = async () => { + connection.closed = true + return proof.promise + } + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + let stopped: unknown + try { + const stopping = host['tasks'] + .serialize(SESSION, () => host['lifetime'].stopAgent(SESSION, { cause: 'evict' })) + .then( + () => 'proven', + (error: unknown) => error + ) + await vi.advanceTimersByTimeAsync(STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS + 2_000) + stopped = await stopping + } finally { + vi.useRealTimers() + } + // Unverifiable for that caller, and the close still running: nothing claims the child exited. + expect(stopped).toMatchObject({ step: 'stop-provider-child' }) + expect(child()?.close?.attempt).toBeDefined() + expect(lease()?.claimStatus).toBe('live') + + proof.resolve(true) + + await eventually(() => expect(child()).toBeNull()) + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'evict' }) + await eventually(() => expect(lease()?.claimStatus).toBe('released')) +}) + +it('ends the record when the root exits after its close gave up, with no one asking again', async () => { + const connection = await stopWithUnprovenClose(1) + + // The connection reports the root's exit as the end of the close Orca began. + connection.handlers.onExit?.(new Error('claude exited'), { expected: true }) + + await eventually(() => expect(child()).toBeNull()) + await eventually(() => expect(lease()?.claimStatus).toBe('released')) + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'user-stop' }) + expect(connection.closeCount).toBe(2) + expect(claude.connections).toHaveLength(1) +}) + +it('sends after a proven exit whose resume-point write and lease release both failed', async () => { + const connection = claude.connections[0]! + await openTurn(connection) + refuseWritesOnceClosed(connection) + persistHandle.mockRejectedValueOnce(new Error('resume point not written')) + const transition = store.transitionHandoff.bind(store) + vi.spyOn(store, 'transitionHandoff') + .mockImplementationOnce(async () => { + throw new Error('store unavailable') + }) + .mockImplementation(transition) + await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) + frame(connection, INTERRUPTED_RESULT) + await eventually(() => expect(child()).toBeNull()) + // Both reported; neither kept the dead child on record. + await eventually(() => + expect(scopes()).toEqual( + expect.arrayContaining(['provider-close-after-exit', 'exit-wind-down']) + ) + ) + expect(lease()?.claimStatus).toBe('live') + + // The start writes the release from this host's proof of that exit. + await send('Carry on.') + await resumedWith(connection, 'Carry on.') +}) + +it('rejects a message whose start meets a close still unverifiable, and starts nothing beside it', async () => { + const connection = await stopWithUnprovenClose(2) + + const next = await send('Carry on.') + + await eventually(async () => expect((await submission(next))?.dispatchState).toBe('rejected')) + const rejected = (await submission(next))! + expect(rejected.reason).toBe( + "Orca couldn't confirm Claude's previous process ended. Send your message to try again." + ) + expect(rejected.rejection).toMatchObject({ + kind: 'restartFailed', + refusal: { details: { reason: 'previousExitUnverifiable' } } + }) + // Rejected, never held: no waiting note, and the old child still on record. + const notes = (await host.journalSnapshot(SESSION)).items.filter( + (item) => item.body.kind === 'status' && item.body.failure?.kind === 'previousExitUnverifiable' + ) + expect(notes).toEqual([]) + expect(connection.closeCount).toBe(2) + expect(claude.connections).toHaveLength(1) + expect(wrote(connection, 'Carry on.')).toBe(false) + expect(child()?.close).toBeDefined() +}) + +it('refuses an option change while the close stays unverifiable, never writing to the old child', async () => { + const connection = await stopWithUnprovenClose(2) + + await expect(setModel('claude-opus-5')).resolves.toMatchObject({ + ok: false, + refusal: { + code: 'agent_session_ownership_unknown', + details: { reason: 'previousExitUnverifiable', ownerVerdict: 'unverifiable' } + } + }) + expect(connection.closeCount).toBe(2) + expect(connection.calls.some((call) => call.subtype === 'set_model')).toBe(false) + + // Joined again once the exit is proven, the pick is the chat's at rest, for the next start. + await expect(setModel('claude-opus-5')).resolves.toMatchObject({ ok: true }) + expect(child()).toBeNull() + expect(store.getRecord(SESSION)?.options).toMatchObject({ model: 'claude-opus-5' }) +}) + +it('reports a descendant left behind by a proven root exit, and blocks nothing', async () => { + const connection = claude.connections[0]! + await openTurn(connection) + refuseWritesOnceClosed(connection) + connection.close = async () => { + connection.closeCount += 1 + connection.closed = true + connection.exitVerdict = { root: 'exited', tree: 'live' } + return false + } + await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) + frame(connection, INTERRUPTED_RESULT) + + await eventually(() => expect(child()).toBeNull()) + expect(scopes()).toContain('provider-close-after-exit') + await send('Carry on.') + await resumedWith(connection, 'Carry on.') +}) + +it('has quit join a close still unverifiable, and hand the lease back once it proves', async () => { + const connection = await stopWithUnprovenClose(1) + + await host.flushAllStreamedEvents() + + expect(connection.closeCount).toBe(2) + expect(lease()).toMatchObject({ claimStatus: 'released', ownerProcess: null }) +}) + +it('has the idle reaper join a close still unverifiable, ending the record with the Stop it finishes', async () => { + const connection = await stopWithUnprovenClose(1) + + await host['lifetime'].idleSweep.tick() + + expect(connection.closeCount).toBe(2) + expect(lease()).toMatchObject({ claimStatus: 'released', ownerProcess: null }) + expect(host.hasSession(SESSION)).toBe(false) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-stop-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-stop-send.test.ts deleted file mode 100644 index b86e25674641..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-stop-send.test.ts +++ /dev/null @@ -1,569 +0,0 @@ -// A Claude Stop whose close could not prove the child gone, then the user's next message, on the -// shipping adapter and host. That child takes no input, so the send retries the stop first; still -// unproven, the message waits with its reason until a later retry proves the exit. - -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, expect, it, vi } from 'vitest' -import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' -import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' -import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-live-turn' -import { claudeUnwrittenUserMessageError } from '../../claude/claude-agent-sdk-user-message-queue' -import { ClaudeStructuredSessionAdapter } from '../../claude/claude-structured-session-adapter' -import { - fakeClaude, - PROVIDER_SESSION_ID, - type FakeConnection -} from '../../claude/claude-structured-session-test-support' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' -import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' -import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import { StructuredAgentSessionHost } from './structured-agent-session-host' -import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' -import { - HOST_TEST_NOW as NOW, - HOST_TEST_SESSION as SESSION, - hostTestAttachParams, - hostTestMessage, - hostTestOperationId, - resetHostTestOperationIds -} from './structured-agent-session-host-test-data' - -const CALLER = { callerKey: 'client-1' } -const CAPABILITIES = ['interrupt_receipt_v1', 'interrupt_cancel_queued_v1', 'msg_lifecycle_v1'] - -let root: string -let host: StructuredAgentSessionHost -let adapter: ClaudeStructuredSessionAdapter -let store: AgentSessionRecordStore -let claude: ReturnType -let log: ReturnType - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'orca-claude-unproven-stop-send-')) - resetHostTestOperationIds() - log = recordingStructuredAgentSessionLogger() - claude = fakeClaude({ - replayUuid: null, - routes: { interrupt: () => ({ still_queued: [], cancelled: [] }) } - }) - const lifecycle: Promise[] = [] - adapter = new ClaudeStructuredSessionAdapter({ - resolveLaunch: async () => ({ - pathToClaudeCodeExecutable: 'claude', - options: {}, - cwd: root, - claudeConfigDir: join(root, 'claude-home'), - providerSessionId: PROVIDER_SESSION_ID, - resumeLeafUuid: null, - resumesTranscript: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0, - continuesChain: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0 - }), - onEvent: (event) => { - const mapped = structuredClaudeLifecycleEvent(event) - if (mapped) { - lifecycle.push(host.handleAdapterEvent(mapped)) - } - }, - onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), - openConnection: claude.openConnection, - readProcessStartTime: async () => 1_700_000_000_000, - now: () => NOW - }) - store = await openTestAgentSessionRecordStore(root) - host = new StructuredAgentSessionHost({ - store, - adapter: Object.assign(adapter, { supportsCreate: () => true }), - journalDatabase: openTestJournalHostDatabase(root), - claimKeyId: 'key-1', - mintSpawnToken: () => 'spawn-a', - logger: log.logger, - now: () => NOW - }) - const params = hostTestAttachParams(null, { - provider: 'claude', - agent: 'claude', - accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, - providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION_ID, leafUuid: null } - }) - expect(await host.attach(CALLER, params)).toMatchObject({ ok: true }) - await adapter.awaitStarted(SESSION) - await Promise.all(lifecycle) -}) - -afterEach(async () => { - await adapter.closeAll() - await host.flushAllStreamedEvents() - await rm(root, { recursive: true, force: true }) -}) - -function eventually(assertion: () => T | Promise): Promise { - return vi.waitFor(assertion, { timeout: 10_000 }) -} - -function envelope( - method: - | 'agentSession.send' - | 'agentSession.cancel' - | 'agentSession.setOption' - | 'agentSession.queuedMessageSend', - // The fingerprint's own field shape, as the sibling host tests type it. - fields: Parameters[0]['fields'] -) { - return { - sessionId: SESSION, - clientOperationId: hostTestOperationId(), - expectedRuntimeFence: store.getRecord(SESSION)!.lease.runtimeFence, - payloadFingerprint: computeAgentSessionPayloadFingerprint({ - method, - sessionId: SESSION, - fields - }) - } -} - -async function send(text: string): Promise { - const body = hostTestMessage(text) - const sent = await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - if (!sent.ok) { - throw new Error(`send refused: ${JSON.stringify(sent.refusal)}`) - } - return sent.value.clientMessageId -} - -async function submission(clientMessageId: string) { - await host.flushStreamedEvents(SESSION) - return (await host.journalSnapshot(SESSION)).submissions.find( - (entry) => entry.clientMessageId === clientMessageId - ) -} - -function frame(connection: FakeConnection, message: Record): void { - connection.handlers.onMessage?.({ session_id: PROVIDER_SESSION_ID, ...message }) -} - -function wrote(connection: FakeConnection, text: string): boolean { - return connection.sent.some((message) => JSON.stringify(message).includes(text)) -} - -async function openTurn(connection: FakeConnection): Promise { - const text = 'Write a long reply.' - const clientMessageId = await send(text) - await eventually(() => expect(wrote(connection, text)).toBe(true)) - frame(connection, { - type: 'system', - subtype: 'init', - uuid: 'init-1', - model: 'claude-sonnet-5', - capabilities: CAPABILITIES - }) - const written = connection.sent.at(-1)! - frame(connection, { ...written, uuid: written.uuid }) - frame(connection, { - type: 'assistant', - uuid: 'stopped-turn-leaf', - parent_tool_use_id: null, - message: { id: 'msg-1', role: 'assistant', content: [{ type: 'text', text: 'Working on' }] } - }) - await eventually(async () => - expect((await submission(clientMessageId))?.dispatchState).toBe('accepted') - ) - expect( - activeStructuredAgentSessionTurnId((await host.journalSnapshot(SESSION)).items) - ).not.toBeNull() -} - -function stop() { - return host.cancel(CALLER, { envelope: envelope('agentSession.cancel', {}) }) -} - -function laneDrained(): Promise { - return host['tasks'].serialize(SESSION, async () => {}) -} - -/** A commit queues a serialized wake, and the wake queues its delivery step behind that. */ -async function commitSettled(): Promise { - await laneDrained() - await laneDrained() -} - -/** As the real connection: once a close begins it refuses every write, proven or not. */ -function closeUnprovenFor(connection: FakeConnection, failures: number): void { - const close = connection.close - let left = failures - connection.close = async () => { - if (left === 0) { - return close() - } - left -= 1 - connection.closeCount += 1 - connection.closed = true - return false - } - const write = connection.send - connection.send = (message, beforeDispatch) => - connection.closed - ? Promise.reject( - claudeUnwrittenUserMessageError(new Error('claude stream-json connection is closed')) - ) - : write(message, beforeDispatch) -} - -const INTERRUPTED_RESULT = { - type: 'result', - subtype: 'error_during_execution', - is_error: true, - terminal_reason: 'aborted_streaming', - uuid: 'interrupted-result' -} - -/** Stops a turn whose child's close cannot prove the exit `failures` times, and lets the Stop's - * second step fail on it. */ -async function stopWithUnprovenClose(failures: number): Promise { - const connection = claude.connections[0]! - await openTurn(connection) - closeUnprovenFor(connection, failures) - await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) - frame(connection, INTERRUPTED_RESULT) - await laneDrained() - expect(connection.closeCount).toBe(1) - expect(owedWindDown()).toBeDefined() - return connection -} - -function owedWindDown() { - return host['sessions'].get(SESSION)?.owesProviderChildWindDown -} - -async function waitRows() { - await host.flushStreamedEvents(SESSION) - return (await host.journalSnapshot(SESSION)).items.flatMap((item) => - item.body.kind === 'status' && item.body.failure?.kind === 'previousExitUnverifiable' - ? [item.body] - : [] - ) -} - -async function resumedWith(connection: FakeConnection, text: string): Promise { - return eventually(() => { - const started = claude.connections.at(-1)! - expect(started).not.toBe(connection) - expect(wrote(started, text)).toBe(true) - return started - }) -} - -it('retries the close a Stop could not prove before the next message, then sends it to a resumed child', async () => { - const connection = await stopWithUnprovenClose(1) - - const next = await send('Carry on.') - const resumed = await resumedWith(connection, 'Carry on.') - - expect(connection.closeCount).toBe(2) - expect(wrote(connection, 'Carry on.')).toBe(false) - expect(resumed.closed).toBe(false) - expect(owedWindDown()).toBeUndefined() - expect((await submission(next))?.dispatchState).toBe('pending') - expect(await waitRows()).toEqual([]) -}) - -it('holds the message with its reason while the exit stays unverifiable, and sends it once a later retry proves it', async () => { - const connection = await stopWithUnprovenClose(3) - - // Never refused: the send is accepted and returns at once. - const next = await send('Carry on.') - await eventually(async () => expect(await waitRows()).toHaveLength(1)) - await laneDrained() - expect(await waitRows()).toEqual([ - { - kind: 'status', - tone: 'warning', - text: "Orca couldn't confirm Claude's previous process ended. Messages wait to be sent until Orca confirms it has ended.", - failure: { kind: 'previousExitUnverifiable' } - } - ]) - // Still queued, drawn below the chat; never written to the child the Stop could not end. - expect(await submission(next)).toMatchObject({ dispatchState: 'pending' }) - expect((await submission(next))?.handedOverAt).toBeUndefined() - expect(claude.connections).toHaveLength(1) - expect(wrote(connection, 'Carry on.')).toBe(false) - expect(owedWindDown()).toBeDefined() - // The Stop's attempt and the send's one retry: the row's own commit retries nothing. - expect(connection.closeCount).toBe(2) - - // A second message retries once more, and waits under the same row. - const second = await send('And this.') - await eventually(() => expect(connection.closeCount).toBe(3)) - await laneDrained() - expect(await submission(second)).toMatchObject({ dispatchState: 'pending' }) - expect(await waitRows()).toHaveLength(1) - - // The sweep's next tick retries the stop; the exit is proven and both messages go out. - await host['lifetime'].idleSweep.tick() - const resumed = await resumedWith(connection, 'Carry on.') - await eventually(() => expect(wrote(resumed, 'And this.')).toBe(true)) - expect(connection.closeCount).toBe(4) - expect(owedWindDown()).toBeUndefined() - expect(await waitRows()).toHaveLength(1) -}) - -it('lets a held message be withdrawn with Stop, and the owed stop still ends on the next retry', async () => { - const connection = await stopWithUnprovenClose(2) - const next = await send('Carry on.') - await eventually(async () => expect(await waitRows()).toHaveLength(1)) - await laneDrained() - - await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) - expect(await submission(next)).toMatchObject({ dispatchState: 'rejected' }) - expect(owedWindDown()).toBeDefined() - - await host['lifetime'].idleSweep.tick() - expect(connection.closeCount).toBe(3) - expect(owedWindDown()).toBeUndefined() - expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - // Nothing was waiting, so no child starts. - expect(claude.connections).toHaveLength(1) - expect(wrote(connection, 'Carry on.')).toBe(false) -}) - -function setModel(model: string) { - const fields = { key: 'model', value: model } - return host.setOption(CALLER, { envelope: envelope('agentSession.setOption', fields), ...fields }) -} - -it('refuses an option change with the exit unverifiable after retrying the stop, never writing to the old child', async () => { - const connection = await stopWithUnprovenClose(2) - - await expect(setModel('claude-opus-5')).resolves.toMatchObject({ - ok: false, - refusal: { - code: 'agent_session_ownership_unknown', - details: { reason: 'previousExitUnverifiable', ownerVerdict: 'unverifiable' } - } - }) - expect(connection.closeCount).toBe(2) - expect(connection.calls.some((call) => call.subtype === 'set_model')).toBe(false) - expect(owedWindDown()).toBeDefined() - - // Trying again retries the stop; proven, the pick is the chat's at rest, for the next start. - await expect(setModel('claude-opus-5')).resolves.toMatchObject({ ok: true }) - expect(connection.closeCount).toBe(3) - expect(owedWindDown()).toBeUndefined() - expect(connection.calls.some((call) => call.subtype === 'set_model')).toBe(false) - expect(store.getRecord(SESSION)?.options).toMatchObject({ model: 'claude-opus-5' }) - expect(claude.connections).toHaveLength(1) -}) - -function stopBackgroundTasks() { - const fields = { scope: 'background-tasks' as const } - return host.cancel(CALLER, { envelope: envelope('agentSession.cancel', fields), ...fields }) -} - -/** Holds the user's next message on a Stop whose exit stays unproven through the send's retry. */ -async function heldAfterStop(): Promise { - const connection = await stopWithUnprovenClose(2) - await send('Carry on.') - await eventually(async () => expect(await waitRows()).toHaveLength(1)) - await laneDrained() - expect(connection.closeCount).toBe(2) - return connection -} - -it('sends a held message once an option change proves the exit: the stop that lands hands it over', async () => { - const connection = await heldAfterStop() - - await expect(setModel('claude-opus-5')).resolves.toMatchObject({ ok: true }) - expect(connection.closeCount).toBe(3) - const resumed = await resumedWith(connection, 'Carry on.') - expect(resumed.closed).toBe(false) - expect(owedWindDown()).toBeUndefined() - expect(store.getRecord(SESSION)?.options).toMatchObject({ model: 'claude-opus-5' }) -}) - -it('sends a held message once a background-task stop proves the exit, though that stop finds no agent', async () => { - const connection = await heldAfterStop() - - // Proven gone, the old agent has no tasks left to stop, and the message still goes out. - await expect(stopBackgroundTasks()).resolves.toMatchObject({ - ok: false, - refusal: { details: { reason: 'noLiveOwner' } } - }) - expect(connection.closeCount).toBe(3) - await resumedWith(connection, 'Carry on.') - expect(owedWindDown()).toBeUndefined() -}) - -it('sends a message held after a tab close whose exit was unproven, rather than rejecting it as closed', async () => { - const connection = claude.connections[0]! - closeUnprovenFor(connection, 2) - await expect(host.close(SESSION, 'user-close')).rejects.toThrow() - expect(owedWindDown()).toMatchObject({ cause: 'user-close' }) - - // The chat stays open on the host; the user sends again, and the message waits on that close. - const next = await send('Carry on.') - await eventually(async () => expect(await waitRows()).toHaveLength(1)) - await laneDrained() - expect(connection.closeCount).toBe(2) - - // The close was asked for before this message, so the retry that lands ends nothing it waited on. - await host['lifetime'].idleSweep.tick() - await laneDrained() - expect(await submission(next)).not.toMatchObject({ dispatchState: 'rejected' }) - await resumedWith(connection, 'Carry on.') - expect(owedWindDown()).toBeUndefined() -}) - -it('never stops a live child for a wind-down another, earlier child still owes', async () => { - const connection = claude.connections[0]! - const session = host['sessions'].get(SESSION)! - session.owesProviderChildWindDown = { - generation: 'an-earlier-child', - fence: session.child!.fence, - cause: 'user-stop', - requestedAt: session.journal.cursor() - } - - await send('Carry on.') - await eventually(() => expect(wrote(connection, 'Carry on.')).toBe(true)) - await laneDrained() - await host['lifetime'].idleSweep.tick() - expect(connection.closeCount).toBe(0) - expect(claude.connections).toHaveLength(1) -}) - -it('retries once per new message: commits after a second waiting message retry nothing', async () => { - const connection = await stopWithUnprovenClose(6) - await send('Carry on.') - await eventually(async () => expect(await waitRows()).toHaveLength(1)) - await laneDrained() - await send('And this.') - await eventually(() => expect(connection.closeCount).toBe(3)) - await laneDrained() - - // Any journal commit wakes the delivery loop while a message is queued. - const session = host['sessions'].get(SESSION)! - for (const n of [1, 2, 3]) { - await session.journal.appendItem( - { provider: 'orca', clientMessageId: `unrelated-${n}` }, - { kind: 'status', text: `Unrelated ${n}.` }, - { fence: store.getRecord(SESSION)!.lease.runtimeFence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } - ) - await commitSettled() - } - expect(connection.closeCount).toBe(3) - expect(claude.connections).toHaveLength(1) - - // The sweep still retries, one pass a tick, until the exit is proven; then both go out. - for (const _tick of [1, 2, 3, 4]) { - await host['lifetime'].idleSweep.tick() - } - expect(connection.closeCount).toBe(7) - const resumed = await resumedWith(connection, 'Carry on.') - await eventually(() => expect(wrote(resumed, 'And this.')).toBe(true)) -}) - -it('queues a follow-up as a draft by default while a message waits, and Steer retries the stop once for it', async () => { - const connection = await stopWithUnprovenClose(3) - await send('Carry on.') - await eventually(async () => expect(await waitRows()).toHaveLength(1)) - await laneDrained() - - // Queueing follow-ups is the default: the waiting message reads as working, so this is a draft. - const body = hostTestMessage('And this.') - const delivery = 'queue-if-active' as const - const queued = await host.send(CALLER, { - envelope: envelope('agentSession.send', { body, delivery }), - body, - delivery - }) - expect(queued).toMatchObject({ ok: true, value: { queued: { state: 'waiting' } } }) - await commitSettled() - expect(connection.closeCount).toBe(2) - - // Steer makes it a waiting message, which retries once and waits under the same note. - const messageId = queued.ok && 'queued' in queued.value ? queued.value.queued.messageId : '' - await expect( - host.queuedMessageSend(CALLER, { - envelope: envelope('agentSession.queuedMessageSend', { messageId }), - messageId - }) - ).resolves.toMatchObject({ ok: true }) - await eventually(() => expect(connection.closeCount).toBe(3)) - await commitSettled() - expect(connection.closeCount).toBe(3) - expect(claude.connections).toHaveLength(1) - expect(await waitRows()).toHaveLength(1) - - await host['lifetime'].idleSweep.tick() - const resumed = await resumedWith(connection, 'Carry on.') - await eventually(() => expect(wrote(resumed, 'And this.')).toBe(true)) -}) - -it('rejects as closed a message a second tab close closed, though that close could not reject it itself', async () => { - const connection = claude.connections[0]! - closeUnprovenFor(connection, 3) - await expect(host.close(SESSION, 'user-close')).rejects.toThrow() - const next = await send('Carry on.') - await eventually(async () => expect(await waitRows()).toHaveLength(1)) - await laneDrained() - expect(connection.closeCount).toBe(2) - - // The second close's own rejection of what is queued fails; its stop is a new ask all the same. - const session = host['sessions'].get(SESSION)! - vi.spyOn(session.journal, 'rejectQueuedSubmissions').mockRejectedValueOnce(new Error('disk full')) - await expect(host.close(SESSION, 'user-close')).rejects.toThrow() - expect(connection.closeCount).toBe(3) - expect(await submission(next)).toMatchObject({ dispatchState: 'pending' }) - - // The retry that proves the exit closes what that second close closed. - await host['lifetime'].idleSweep.tick() - await commitSettled() - expect(connection.closeCount).toBe(4) - expect(await submission(next)).toMatchObject({ dispatchState: 'rejected' }) - expect(claude.connections).toHaveLength(1) -}) - -it('notes why a message waits when another operation failed its retry before the message was delivered', async () => { - const connection = await stopWithUnprovenClose(2) - - // The send is accepted, then the option change's failed retry runs before the send's delivery. - const sent = send('Carry on.') - const option = setModel('claude-opus-5') - await expect(option).resolves.toMatchObject({ - ok: false, - refusal: { details: { reason: 'previousExitUnverifiable' } } - }) - await sent - await commitSettled() - expect(connection.closeCount).toBe(2) - expect(await waitRows()).toHaveLength(1) - expect(claude.connections).toHaveLength(1) -}) - -it('keeps an option change at rest when the Stop proved the exit and only its bookkeeping keeps failing', async () => { - const connection = claude.connections[0]! - await openTurn(connection) - // The close proves the exit; draining what the old agent wrote fails for the Stop. - const barrierLost = { ok: false as const, error: new Error('drain barrier lost') } - const drained = vi - .spyOn(host['runtimeState'].eventSinkFor(SESSION), 'drained') - .mockResolvedValueOnce(barrierLost) - .mockResolvedValueOnce(barrierLost) - await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) - frame(connection, INTERRUPTED_RESULT) - await laneDrained() - expect(connection.closeCount).toBe(1) - expect(host['sessions'].get(SESSION)?.child).toBeNull() - expect(owedWindDown()).toBeDefined() - - // The option change's retry of that bookkeeping fails again: reported, and the pick still lands. - drained.mockResolvedValueOnce(barrierLost) - await expect(setModel('claude-opus-5')).resolves.toMatchObject({ ok: true }) - expect(owedWindDown()).toBeDefined() - expect(log.scopes().filter((scope) => scope === 'owed-stop-retry')).toHaveLength(1) - expect(store.getRecord(SESSION)?.options).toMatchObject({ model: 'claude-opus-5' }) - expect(connection.closeCount).toBe(1) - expect(claude.connections).toHaveLength(1) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts index 33ef8080c030..491c4a419723 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts @@ -253,7 +253,7 @@ describe('a turn cut short by closing its provider', () => { ).toMatchObject({ key: 'failedAfter' }) }) - it("keeps the user's cancellation when a close aborts after the provider settled, then retries", async () => { + it("keeps the user's cancellation when a close's drain fails after the provider settled", async () => { await runningTurn() const sink = host['runtimeState'].eventSinkFor(SESSION) const drained = sink.drained.bind(sink) @@ -265,10 +265,9 @@ describe('a turn cut short by closing its provider', () => { } return drained() }) - // The adapter settled the turn, then the close aborted at the drain after it. - await expect(host.close(SESSION, 'user-close')).rejects.toThrow() + // The adapter settled the turn, then the drain after it failed: reported, and the close ends. + await expect(host.close(SESSION, 'user-close')).resolves.toBeUndefined() expect(closeCalls).toBe(1) - await host.close(SESSION, 'user-close') const { turn } = await settledTurn() expect(turn).toMatchObject({ state: 'interrupted', outcome: 'cancellation' }) }) @@ -277,7 +276,7 @@ describe('a turn cut short by closing its provider', () => { ['user-close', { outcome: 'cancellation' }], ['evict', { outcome: undefined }] ] as const)( - "keeps a %s's cause when the idle sweep finishes a wind-down it could not", + "keeps a %s's cause when the close's drain fails before the host settles its turn", async (cause, verdict) => { providerEnd = null await runningTurn() @@ -291,10 +290,8 @@ describe('a turn cut short by closing its provider', () => { } return drained() }) - // The provider is proven gone, then the close aborts before the host settles its turn. - await expect(host.close(SESSION, cause)).rejects.toThrow() - - await host.collaboratorsForTests().lifetime.idleSweep.tick() + // The provider is proven gone, then the drain fails: the host still settles the turn. + await expect(host.close(SESSION, cause)).resolves.toBeUndefined() expect(closeCalls).toBe(1) const { turn } = await settledTurn() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts index ef254fdc765c..6f7ef7ee7d1b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts @@ -58,12 +58,12 @@ beforeEach(async () => { return {} } const store = await openTestAgentSessionRecordStore(root) - // The runtime's wiring: an echo accepts its send, and an exit reaches the host. + // The runtime's wiring: an echo accepts its send, and every exit reaches the host. launch = {} const adapter = adapterFor(codex, launch, [], { onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), onEvent: (event) => { - if (event.type === 'ended' && 'cause' in event && event.cause === 'unexpected-exit') { + if (event.type === 'ended' && 'cause' in event) { void host.handleAdapterEvent(event) } } @@ -448,16 +448,17 @@ describe('a Codex Stop whose interrupt failed', () => { }) describe('a message after a Codex Stop whose exit was unproven', () => { - it('retries that stop first, then goes to a fresh Codex, never to the old one', async () => { + /** A Stop whose interrupt fails ends the child, whose close cannot prove the exit `failures` + * times. As the real connection, it refuses every request once a close begins. */ + async function stopWithUnprovenClose(failures: number) { await runningTurn() codex.routes['turn/interrupt'] = () => { throw interruptFailure('internal error') } - // As the real connection: once a close begins it refuses every request, proven or not. const old = codex.connections.at(-1)! const close = old.close const request = old.request - let unproven = 1 + let unproven = failures old.close = async () => { old.closed = true if (unproven === 0) { @@ -470,28 +471,66 @@ describe('a message after a Codex Stop whose exit was unproven', () => { old.closed ? Promise.reject(new Error('codex app-server is closing')) : request(method, params) - await stop() await host.flushStreamedEvents(SESSION) - expect(host['sessions'].get(SESSION)?.owesProviderChildWindDown).toBeDefined() + return old + } + + const startedWith = (connection: (typeof codex.connections)[number], text: string) => + connection.calls.some( + (call) => call.method === 'turn/start' && JSON.stringify(call.params).includes(text) + ) + + it('rejects the message while the close stays unverifiable, and starts no Codex beside it', async () => { + const old = await stopWithUnprovenClose(2) + + const sent = await send('carry on') + if (!sent.ok || !('submission' in sent.value)) { + throw new Error(JSON.stringify(sent)) + } + const id = sent.value.submission.clientMessageId + await vi.waitFor(async () => + expect( + (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === id + ) + ).toMatchObject({ + dispatchState: 'rejected', + reason: + "Orca couldn't confirm Codex's previous process ended. Send your message to try again." + }) + ) + expect(codex.connections).toHaveLength(1) + expect(startedWith(old, 'carry on')).toBe(false) + expect(host['sessions'].get(SESSION)?.child?.close).toBeDefined() + }) + + it('ends the record when the root exits after its close gave up, with no one asking again', async () => { + const old = await stopWithUnprovenClose(1) + const handled = vi.spyOn(host, 'handleAdapterEvent') + + old.handlers.onExit?.(new Error('codex app-server exited'), { expected: true }) + + // The close Orca began, ended: Orca's, never blamed on Codex as a crash. + expect(handled).toHaveBeenCalledWith( + expect.objectContaining({ cause: 'requested-close', failure: { kind: 'hostFault' } }) + ) + await vi.waitFor(() => expect(host['sessions'].get(SESSION)?.child).toBeNull()) + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'user-stop' }) + expect(codex.connections).toHaveLength(1) + }) + + it('joins that close first, then goes to a fresh Codex, never to the old one', async () => { + const old = await stopWithUnprovenClose(1) + expect(host['sessions'].get(SESSION)?.child?.close).toMatchObject({ cause: 'user-stop' }) // The next start resumes the chat's thread, as the runtime's launch resolves it from the record. launch.resumeThreadId = THREAD const sent = await send('carry on') expect(sent).toMatchObject({ ok: true }) await vi.waitFor(() => expect(codex.connections).toHaveLength(2)) - await vi.waitFor(() => - expect( - codex.connections[1]!.calls.some( - (call) => call.method === 'turn/start' && JSON.stringify(call.params).includes('carry on') - ) - ).toBe(true) - ) - expect( - old.calls.some( - (call) => call.method === 'turn/start' && JSON.stringify(call.params).includes('carry on') - ) - ).toBe(false) - expect(host['sessions'].get(SESSION)?.owesProviderChildWindDown).toBeUndefined() + await vi.waitFor(() => expect(startedWith(codex.connections[1]!, 'carry on')).toBe(true)) + expect(startedWith(old, 'carry on')).toBe(false) + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'user-stop' }) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts index c084d2b2e07c..72aafc92baf2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts @@ -19,7 +19,6 @@ import { sweepOnce, type RestTestRig } from './structured-agent-session-rest-test-rig' -import { StructuredAgentSessionIdleSweep } from './structured-agent-session-idle-sweep' import { hostTestAttachParams } from './structured-agent-session-host-test-data' let rig: RestTestRig @@ -57,9 +56,8 @@ describe('a stop that fails', () => { rig.clock.now += IDLE_MS + 1 await sweepOnce(rig.host) - expect(openSession()?.owesProviderChildWindDown).toMatchObject({ - generation: expect.any(String) - }) + // The child stays on record with its close unproven, and the next tick joins it. + expect(openSession()?.child?.close).toMatchObject({ cause: 'evict' }) await sweepOnce(rig.host) expect(rig.adapter.closeSession).toHaveBeenCalledTimes(2) @@ -67,23 +65,21 @@ describe('a stop that fails', () => { expect(statusRows().at(-1)?.hostExecutionOwned).toBeUndefined() }) - it('finishes its wind-down on the next tick, whatever its own writes did to the clock (P2-11 b)', async () => { + it('re-derives a release its wind-down could not write before the handle closes (P2-11 b)', async () => { await foundRestTestChat(rig) rig.clock.now += IDLE_MS + 1 - // The child is proven gone and its work settled, then handing the lease back fails. + // The child is proven gone and its work settled, then handing the lease back fails once. const transition = vi .spyOn(rig.store, 'transitionHandoff') .mockRejectedValueOnce(new Error('store unavailable')) await sweepOnce(rig.host) - expect(transition).toHaveBeenCalledOnce() - expect(rig.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') - // Five minutes later, not thirty. - rig.clock.now += 5 * 60_000 - await sweepOnce(rig.host) + // Reported, never retried as a stop: the handle's close writes the release from the proof. + expect(transition).toHaveBeenCalledTimes(2) expect(rig.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') expect(rig.adapter.closeSession).toHaveBeenCalledOnce() + expect(rig.host.hasSession(SESSION)).toBe(false) }) }) @@ -287,50 +283,3 @@ describe('a start that never finishes (P2-15)', () => { await expect(sweepOnce(rig.host)).resolves.toBeUndefined() }) }) - -describe('the wind-down retry with a message queued (P2-31)', () => { - it('waits for the delivery rather than rejecting a message accepted after the failed stop', async () => { - const queued = { clientMessageId: 'm', dispatchState: 'pending', handoverRecorded: true } - const session = { - journal: { - submissions: () => [queued], - pendingSubmissions: () => [queued], - snapshot: () => ({ items: [] }) - }, - child: null, - owesProviderChildWindDown: { generation: 'generation-1', fence: 1 } - } - const stopAgent = vi.fn(async () => undefined) - const finishOwedWindDown = vi.fn(async () => true) - const sweep = new StructuredAgentSessionIdleSweep({ - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a session fixture carrying only the journal and child facts the sweep reads. - sessions: Object.assign(new Map([[SESSION, session as never]]), { - lastActivityAt: () => 0, - touch: () => undefined - }), - serialize: (_id, task) => task(), - now: () => IDLE_MS + 1, - isDisposed: () => false, - deliveryActive: () => true, - childWork: () => undefined, - hasOpenDispatch: () => false, - providerHoldsDispatch: () => false, - stopAgent, - stopStartingAgent: stopAgent, - finishOwedWindDown, - closeConversation: vi.fn(async () => false), - // A failed step fails the test. - logger: { - warn: (_message, fields) => { - throw fields.error - }, - error: (_message, fields) => { - throw fields.error - } - } - }) - await sweep.tick() - expect(stopAgent).not.toHaveBeenCalled() - expect(finishOwedWindDown).not.toHaveBeenCalled() - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 19af6d048d45..3a35b4b138ae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -12,10 +12,10 @@ import { import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import { createJournalOpenReadRefusals } from '../agent-session-journal/journal-open-failure' import type { StructuredAgentSessionConversations } from './structured-agent-session-conversations' +import { releaseLeaseOfEndedStructuredAgentSessionChild } from './structured-agent-session-child-close' import { abandonQueuedStructuredAgentSessionMessages, closeStructuredAgentSessionConversationUnderSerialize, - finishOwedStructuredAgentSessionWindDownUnderSerialize, stopStructuredAgentSessionAgentUnderSerialize, type StructuredAgentSessionCloseCause, type StructuredAgentSessionLifetimeContext, @@ -57,11 +57,13 @@ export function createStructuredAgentSessionConversationLifetime(host: { }) const stopAgent = (sessionId: string, ending: StructuredAgentSessionStopEnding) => stopStructuredAgentSessionAgentUnderSerialize(host.context(), sessionId, ending) - const finishOwedWindDown = (sessionId: string) => - finishOwedStructuredAgentSessionWindDownUnderSerialize(host.context(), sessionId) - const closeConversation = (sessionId: string): Promise => - closeStructuredAgentSessionConversationUnderSerialize( + const closeConversation = async (sessionId: string): Promise => { + // The handle carries the proof that releases a lease its child's wind-down could not. + if (await releaseLeaseOfEndedStructuredAgentSessionChild(host.context(), sessionId)) { + return false + } + return closeStructuredAgentSessionConversationUnderSerialize( { sessions, closeStatus: (id) => { @@ -72,6 +74,7 @@ export function createStructuredAgentSessionConversationLifetime(host: { }, sessionId ) + } const idleSweep = new StructuredAgentSessionIdleSweep({ sessions, @@ -87,7 +90,6 @@ export function createStructuredAgentSessionConversationLifetime(host: { providerHoldsDispatch: (sessionId) => deps().adapter.holdsDispatch?.(sessionId) === true, // The host puts an idle agent to rest: a turn it cuts short is news, not the user's Stop. stopAgent: (sessionId) => stopAgent(sessionId, { cause: 'evict', resting: true }), - finishOwedWindDown, // A host stop: the delivery loop waiting on this child writes the one error row and rejects // what is queued with it, both worded from the hostStopped fact. stopStartingAgent: (sessionId) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-stop.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-stop.test.ts index 155837a01e16..1b0b9310b2be 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-stop.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-stop.test.ts @@ -307,9 +307,10 @@ describe('a Stop that names no turn', () => { expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) expect(closeSession).toHaveBeenCalledExactlyOnceWith(SESSION) + // Bookkeeping after the proven exit: reported, never the Stop's failure. expect(log.entries).toContainEqual( expect.objectContaining({ - fields: expect.objectContaining({ scope: 'stop-child', sessionId: SESSION }) + fields: expect.objectContaining({ scope: 'exit-wind-down', sessionId: SESSION }) }) ) expect(await statusRows()).toEqual(['Cancellation requested.']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts index 0414836f4a5e..3727040e3c0b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -26,10 +26,7 @@ import { structuredAgentSessionStartFailure, type StructuredAgentSessionStartFailureCause } from './structured-agent-session-failure-text' -import { - isStructuredAgentSessionPreviousExitUnverifiable, - type StructuredAgentSessionResumeOutcome -} from './structured-agent-session-agent-start' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-agent-start' import type { StructuredAgentSessionChildEndCause, StructuredAgentSessionEndedChild, @@ -42,10 +39,6 @@ import { } from './structured-agent-session-start-failure-row' import { failedProviderChildStart } from './structured-agent-session-provider-child' import { handOverSubmission } from './structured-agent-session-turns' -import { - recordStructuredAgentSessionWindDownWait, - structuredAgentSessionWindDownWaitHolds -} from './structured-agent-session-wind-down-wait-row' import { structuredAgentSessionCommandRunning } from './structured-agent-session-command-turn' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' @@ -189,24 +182,11 @@ export class StructuredAgentSessionDeliveryLoop { if (!oldest || (session.child && structuredAgentSessionCommandRunning(session.journal))) { return this.stop(sessionId) } - // Already waiting on a stop that could not prove its child gone: a new message retries it, and - // any other retry that lands wakes this loop itself, so the waiting row's own commit does not. - // Another operation's retry may have failed first, so the row is made sure of here too. - if (structuredAgentSessionWindDownWaitHolds(session)) { - await recordStructuredAgentSessionWindDownWait(session, sessionId, this.deps) - return this.stop(sessionId) - } const failedStart = startThatFailedWhileQueued(session, oldest) if (failedStart) { return this.fail(sessionId, failedStart) } const ready = await this.deps.ensureProviderChild(sessionId, oldest.clientMessageId) - if (!ready.ok && isStructuredAgentSessionPreviousExitUnverifiable(ready.refusal)) { - // The start retried that stop first and still could not prove the exit: the message waits, - // saying why, rather than being refused. The row is written once per unproven child. - await recordStructuredAgentSessionWindDownWait(session, sessionId, this.deps) - return this.stop(sessionId) - } if (!ready.ok) { return ready } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 2b9340859f42..1a7742d3e5ff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -8,6 +8,8 @@ import type { import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' import { settleStructuredAgentSessionProviderStarted } from './structured-agent-session-provider-started' import { settleUnexpectedStructuredAgentSessionExit } from './structured-agent-session-unexpected-exit' +import { endClosedStructuredAgentSessionChild } from './structured-agent-session-child-close' +import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' export class StructuredAgentSessionEventRecovery { private readonly sinkFailures = new Set() @@ -22,11 +24,17 @@ export class StructuredAgentSessionEventRecovery { publishStatus?: (sessionId: string) => void serialize: (sessionId: string, task: () => Promise) => Promise now: () => number + /** Where a close the host asked for ends the child's record. */ + lifetime: () => StructuredAgentSessionLifetimeContext } ) {} private get exitContext() { - return { ...this.context, logger: this.context.deps.logger } + return { + ...this.context, + logger: this.context.deps.logger, + wakeDelivery: this.context.lifetime().wakeDelivery + } } recoverAfterSinkFailure(sessionId: string, error: unknown): void { @@ -72,12 +80,16 @@ export class StructuredAgentSessionEventRecovery { .finally(() => this.sinkFailures.delete(sessionId)) } - /** An exit is settled and shown; nothing restarts the child. The next send does, through the - * delivery loop, which also owns any message still queued. */ + /** Every child's exit ends its record here, expected or not. An exit is settled and shown; + * nothing restarts the child. The next send does, through the delivery loop, which also owns any + * message still queued. */ async handle(event: StructuredAgentSessionLifecycleEvent): Promise { if (event.type === 'started') { return settleStructuredAgentSessionProviderStarted(this.context, event) } + if (event.cause === 'requested-close') { + return endClosedStructuredAgentSessionChild(this.context.lifetime(), event) + } await settleUnexpectedStructuredAgentSessionExit(this.exitContext, event) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts index f34caba3593d..12b5316ef6f7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts @@ -1,12 +1,9 @@ -// A bound on how long teardown may take, expressed as a wrapper around the eviction steps rather -// than a change to them. +// A bound on how long a caller waits on teardown, never on the teardown itself. // -// The step list is ordered and abort-on-failure for reasons that have nothing to do with time, and -// a deadline must not disturb either. Wrapping each step's `run` keeps the order, and a timeout -// surfaces as that step failing — which is exactly the behavior wanted here: the rest of the -// eviction aborts, the session stays indexed, and the child stays LOADED. A stuck app-server that -// is still holding a conversation is a better outcome than one killed out from under it; the next -// close retries. +// Two uses. A caller joining a child's close waits this long for the exit's proof and then answers +// `unverifiable`; the close keeps running, and a proof that lands later still ends the record. Each +// wind-down step after the exit is bounded the same way, so one hung step is reported and the rest +// still run. import type { StructuredAgentSessionEvictionStep } from './structured-agent-session-eviction' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts index d12a6509c747..269ced39ac59 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts @@ -5,10 +5,6 @@ import { STRUCTURED_AGENT_SESSION_EVICTION_STEPS, type StructuredAgentSessionEvictionContext } from './structured-agent-session-eviction' -import { - AgentSessionAcquisitionRootExitObservedError, - AgentSessionPreSpawnError -} from './structured-agent-session-adapter' import { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' @@ -27,12 +23,6 @@ function context(): StructuredAgentSessionEvictionContext & { order: string[] } }), close: vi.fn(() => order.push('close')) } as unknown as StructuredAgentSessionEvictionContext['eventSink'], - adapter: { - closeSession: vi.fn(async () => { - order.push('closeSession') - return true - }) - } as unknown as StructuredAgentSessionEvictionContext['adapter'], acknowledgeRelease: vi.fn(() => { order.push('acknowledgeRelease') }), @@ -55,12 +45,11 @@ function runtimeState(): StructuredAgentSessionHostRuntimeState { } as never) } -describe('structured agent session eviction', () => { - it('stops the child before it lets the sink go, then acknowledges the release', async () => { +describe('the wind-down after a proven exit', () => { + it('drains what the child said before it lets the sink go, then acknowledges the release', async () => { const ctx = context() await evictStructuredAgentSession(ctx) expect(ctx.order).toEqual([ - 'closeSession', 'drained', 'settleWork', 'unbind', @@ -71,24 +60,8 @@ describe('structured agent session eviction', () => { ]) }) - it('uses disposal rather than handoff close when the chat is removed', async () => { - const ctx = context() - const closeSession = vi.fn(async () => { - throw new Error('resume cursor unavailable') - }) - const disposeSession = vi.fn(async () => true) - ctx.adapter = { ...ctx.adapter, closeSession, disposeSession } - - await evictStructuredAgentSession(ctx) - - expect(disposeSession).toHaveBeenCalledWith('session-1') - expect(closeSession).not.toHaveBeenCalled() - }) - - it('names every step, so a half-finished eviction says which one failed', () => { + it('names every step, so a failure says which one it was', () => { expect(STRUCTURED_AGENT_SESSION_EVICTION_STEPS.map((step) => step.name)).toEqual([ - 'snapshot-before-stop', - 'stop-provider-child', 'drain-published', 'settle-dead-generation', 'stop-publishing', @@ -99,94 +72,21 @@ describe('structured agent session eviction', () => { ]) }) - it('still stops the child when the pre-stop snapshot cannot drain the sink', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - try { - const ctx = context() - const snapshot = vi.fn() - ctx.beforeProviderChildStop = snapshot - // A journal write that never settles, then a healthy sink once the child is stopped. - vi.mocked(ctx.eventSink.drained).mockReturnValueOnce(new Promise(() => {})) - - const eviction = evictStructuredAgentSession(ctx) - await vi.advanceTimersByTimeAsync(1_000) - - expect(snapshot).toHaveBeenCalledOnce() - expect(ctx.adapter.closeSession).toHaveBeenCalledOnce() - await eviction - } finally { - vi.useRealTimers() - } - }) - - it('aborts after a failed drain barrier without unbinding or acknowledging the release', async () => { - const ctx = context() + // Bookkeeping for a process already gone: none of it may keep the child on record. + it('reports a failed step with its name and still runs every step after it', async () => { + const recorded = recordingStructuredAgentSessionLogger() + const ctx = { ...context(), logger: recorded.logger } ctx.eventSink.drained = vi.fn(async () => { ctx.order.push('drained') return { ok: false, error: new Error('append failed') } }) as unknown as StructuredAgentSessionEvictionContext['eventSink']['drained'] - - await expect(evictStructuredAgentSession(ctx)).rejects.toMatchObject({ - step: 'drain-published' + ctx.releaseLease = vi.fn(async () => { + ctx.order.push('releaseLease') + throw new Error('store unavailable') }) - expect(ctx.eventSink.unbind).not.toHaveBeenCalled() - expect(ctx.eventSink.close).not.toHaveBeenCalled() - expect(ctx.discardSink).not.toHaveBeenCalled() - expect(ctx.releaseLease).not.toHaveBeenCalled() - expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() - expect(ctx.order).toEqual(['closeSession', 'drained']) - }) -}) -// Closing the codex child is not silent: the adapter emits its `ended` event and flushes coalesced -// text as it shuts down, and those rows are what clear the running-turn marker. If the sink is -// already closed the journal keeps claiming the agent is working, forever. -describe('rows the provider emits while closing', () => { - it('still reach the journal', async () => { - const state = runtimeState() - const sessionId = 'session-closing-rows' - const sink = state.eventSinkFor(sessionId) - const published: string[] = [] - sink.bind({ journal: {} as never, fence: 1, publish: () => published.push('final-flush') }) + await expect(evictStructuredAgentSession(ctx)).resolves.toBeUndefined() - await evictStructuredAgentSession({ - sessionId, - logger: recordingStructuredAgentSessionLogger().logger, - eventSink: sink, - adapter: { - closeSession: async () => { - // What codex-structured-session-close does on its way out. - sink.sink.publish() - return true - } - } as never, - discardSink: () => state.discardEventSink(sessionId), - releaseLease: async () => {}, - acknowledgeRelease: () => {} - }) - - expect(published).toEqual(['final-flush']) - }) -}) - -// `closeSession` returning false means the adapter could not prove the child exited and has kept -// the session indexed on purpose so a retry can reach it. -describe('a child that will not stop', () => { - it.each([ - new AgentSessionAcquisitionRootExitObservedError(new Error('root exited')), - new AgentSessionPreSpawnError(new Error('spawn failed')) - ])('continues eviction after an actionable provider verdict', async (error) => { - const ctx = context() - ctx.adapter.closeSession = vi.fn(async () => { - throw error - }) - const stopped = vi.fn() - ctx.onProviderChildStopped = stopped - - await evictStructuredAgentSession(ctx) - - // The host ends its child on the one reading of the verdict, not a second one of its own. - expect(stopped).toHaveBeenCalledWith({ rootGone: true }) expect(ctx.order).toEqual([ 'drained', 'settleWork', @@ -196,31 +96,11 @@ describe('a child that will not stop', () => { 'releaseLease', 'acknowledgeRelease' ]) - }) - - it('aborts without acknowledging the release, so the next stop is a real retry', async () => { - const ctx = context() - ctx.adapter.closeSession = vi.fn(async () => false) - - await expect(evictStructuredAgentSession(ctx)).rejects.toMatchObject({ - step: 'stop-provider-child' - }) - expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() - expect(ctx.discardSink).not.toHaveBeenCalled() - expect(ctx.order).toEqual([]) - }) - - it('reports the failing step and leaves the sink usable for the retry', async () => { - const ctx = context() - ctx.adapter.closeSession = vi.fn(async () => { - throw new Error('child would not stop') - }) - - await expect(evictStructuredAgentSession(ctx)).rejects.toBeInstanceOf( - StructuredAgentSessionEvictionError - ) - expect(ctx.eventSink.close).not.toHaveBeenCalled() - expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() + expect(recorded.entries.map((entry) => entry.fields.error)).toEqual([ + expect.objectContaining({ step: 'drain-published' }), + expect.objectContaining({ step: 'release-lease' }) + ]) + expect(recorded.entries[0]?.fields.error).toBeInstanceOf(StructuredAgentSessionEvictionError) }) }) @@ -235,8 +115,8 @@ describe('eviction against the real sink cache', () => { sessionId, logger: recordingStructuredAgentSessionLogger().logger, eventSink: state.eventSinkFor(sessionId), - adapter: { closeSession: async () => true } as never, discardSink: () => state.discardEventSink(sessionId), + settleWork: async () => {}, releaseLease: async () => {}, acknowledgeRelease: () => {} }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts index dcc31c451d2f..e0743e8fb028 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts @@ -1,51 +1,34 @@ -// Stopping one structured session's provider child and handing its lease back. +// What a host does once its provider child is gone: the bookkeeping that hands the session back. // // Teardown is a DATA list, not a method body, for the reason this file exists at all: the host // tracked which sessions were live in a map, and tore them down at three unrelated call sites // (app quit, handoff to a TUI, and error cleanup). Closing a chat was never wired to any of them, // so a provider child outlived the chat that owned it for the whole app session. // -// ORDER. The provider child stops FIRST. Closing it is not silent: the codex adapter emits its -// `ended` event and flushes coalesced text as part of shutting down, and those are the rows that -// clear the running-turn marker. Draining or closing the sink ahead of that drops them, which -// leaves the durable journal claiming the agent is still working — a worse outcome than the leak -// this teardown exists to fix. So: stop the child, drain what it emitted on its way out, then let -// the sink go. The one step ahead of the stop only reads, for quit's resume offer. +// ORDER. These run only after the child's exit is proven (`structured-agent-session-child-close`). +// Closing it is not silent: the codex adapter emits its `ended` event and flushes coalesced text as +// part of shutting down, and those are the rows that clear the running-turn marker. Draining or +// closing the sink ahead of that drops them, which leaves the durable journal claiming the agent is +// still working. So: drain what the child emitted on its way out, settle, then let the sink go. // -// FAILURE. A step that fails ABORTS the rest. `closeSession` returning false means the child's -// exit was not proven and the adapter has deliberately kept the session indexed so a retry can -// reach it; forgetting it anyway stranded the process forever and reported success. Leaving the -// session in place is what makes the next close a real retry instead of a no-op. +// FAILURE. Each step is bookkeeping for a process already gone, so a step that fails is reported +// and the rest still run: none of them may keep a dead child on record or gate the next send. -import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' -import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' -import type { StructuredAgentSessionStopVerdict } from './structured-agent-session-host-types' import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' export type StructuredAgentSessionEvictionContext = { sessionId: string - hasProviderChild?: boolean eventSink: DeferredStructuredAgentSessionEventSink - adapter: StructuredAgentSessionAdapter logger: StructuredAgentSessionLogger /** Tells the adapter the released lease is done with, so it drops this child's route and index. * The conversation stays: stopping the agent never closes its journal. */ acknowledgeRelease: () => Promise | void /** Drops the cached sink so a later attach mints a fresh one. */ discardSink: () => void - /** Fires right before the stop, while the child's turn and background roster are still live. A - * throw is logged, never allowed to abort the stop. */ - beforeProviderChildStop?: () => void - /** Fires with the stop's verdict once `stopAgentSessionProviderRoot` read the root gone, so host - * bookkeeping stops claiming a child. */ - onProviderChildStopped?: (verdict: StructuredAgentSessionStopVerdict) => void - /** Whether this host still owes the child's wind-down. Distinct from `hasProviderChild`, which a - * proven exit retires mid-run: the two disagree for exactly the steps a retry has to repeat. */ - owesProviderChildWindDown?: boolean /** Settles work owned by the child after its final callbacks have drained. */ - settleWork?: () => Promise + settleWork: () => Promise /** Hands the lease back now that this host's child is proven gone. No-ops when the record is * not this host's to release. */ releaseLease: () => Promise @@ -59,44 +42,25 @@ export type StructuredAgentSessionEvictionStep = { run: (context: StructuredAgentSessionEvictionContext) => Promise | void } +/** Quit's resume offer: what the sidebar shows, read while the child is still running. Events the + * provider already delivered are part of what it showed at the stop. A throw is logged. */ +export async function snapshotBeforeStructuredAgentSessionStop( + context: Pick, + snapshot: () => void +): Promise { + await withTimeout(context.eventSink.drained(), SNAPSHOT_DRAIN_TIMEOUT_MS, null) + try { + snapshot() + } catch { + context.logger.warn('capturing a recovery witness before a stop failed', { + scope: 'recovery-witness', + sessionId: context.sessionId + }) + } +} + export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSessionEvictionStep[] = [ - { - // Quit's resume offer: what the sidebar shows, read while the child is still running. - name: 'snapshot-before-stop', - run: async (context) => { - if (context.hasProviderChild === false || !context.beforeProviderChildStop) { - return - } - // Events the provider already delivered are part of what the sidebar showed at the stop. - await withTimeout(context.eventSink.drained(), SNAPSHOT_DRAIN_TIMEOUT_MS, null) - try { - context.beforeProviderChildStop() - } catch { - context.logger.warn('capturing a recovery witness before a stop failed', { - scope: 'recovery-witness', - sessionId: context.sessionId - }) - } - } - }, - { - name: 'stop-provider-child', - run: async (context) => { - if (context.hasProviderChild === false) { - return - } - // An adapter with no close has nothing to stop; anything else must PROVE the exit. - const stop = context.adapter.disposeSession ?? context.adapter.closeSession - const rootGone = stop - ? await stopAgentSessionProviderRoot(() => stop.call(context.adapter, context.sessionId)) - : true - if (!rootGone) { - throw new Error('provider child exit was not proven') - } - context.onProviderChildStopped?.({ rootGone }) - } - }, { name: 'drain-published', run: async (context) => { @@ -106,11 +70,7 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe } } }, - { - name: 'settle-dead-generation', - run: (context) => - context.owesProviderChildWindDown === false ? undefined : context.settleWork?.() - }, + { name: 'settle-dead-generation', run: (context) => context.settleWork() }, { name: 'stop-publishing', run: (context) => context.eventSink.unbind() }, { name: 'close-sink', run: (context) => context.eventSink.close() }, // Why: the runtime caches one sink per session id and hands the SAME instance to the next @@ -118,10 +78,8 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe // accepts every provider event and publishes none. Attach's own failure path already pairs // these two; eviction has to as well. { name: 'discard-sink', run: (context) => context.discardSink() }, - // Why here and not last: the durable lease still names a process this host just stopped, and a - // record left claiming a live owner is one nothing can resume — the next send would find a - // session it may not acquire. Placed BEFORE the acknowledgement so a release that cannot be - // written aborts while the adapter still routes the session, which is what makes the retry real. + // The durable lease still names a process this host just proved gone; a start re-derives the + // release from that proof if this write fails. { name: 'release-lease', run: (context) => context.releaseLease() }, { name: 'acknowledge-release', run: (context) => context.acknowledgeRelease() } ] @@ -137,11 +95,8 @@ export class StructuredAgentSessionEvictionError extends Error { } } -/** - * Runs the eviction steps in order, stopping at the first failure. The step name travels with the - * error because the caller's only useful response is to retry, and a retry is only safe when the - * session is still indexed — which is exactly what aborting preserves. - */ +/** Runs every wind-down step in order. A failure is reported with the step that failed, and the + * rest still run. */ export async function evictStructuredAgentSession( context: StructuredAgentSessionEvictionContext, steps: readonly StructuredAgentSessionEvictionStep[] = STRUCTURED_AGENT_SESSION_EVICTION_STEPS @@ -150,7 +105,11 @@ export async function evictStructuredAgentSession( try { await step.run(context) } catch (error) { - throw new StructuredAgentSessionEvictionError(step.name, context.sessionId, error) + context.logger.warn('a wind-down step after the agent exited failed', { + scope: 'exit-wind-down', + sessionId: context.sessionId, + error: new StructuredAgentSessionEvictionError(step.name, context.sessionId, error) + }) } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 60c20cb768d9..449c425fb69d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -9,34 +9,26 @@ // reentrant, so every public entry point takes it once and calls these. import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' import { - evictStructuredAgentSession, - STRUCTURED_AGENT_SESSION_EVICTION_STEPS, - type StructuredAgentSessionEvictionContext + snapshotBeforeStructuredAgentSessionStop, + StructuredAgentSessionEvictionError } from './structured-agent-session-eviction' -import { withStructuredAgentSessionEvictionDeadline } from './structured-agent-session-eviction-deadline' +import { joinStructuredAgentSessionChildClose } from './structured-agent-session-child-close' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession, - StructuredAgentSessionOwedWindDown, - StructuredAgentSessionProviderChildIdentity + StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import { - endProviderChild, - pendingProviderChildWindDown, - sameProviderChild, - structuredAgentSessionConversationFence -} from './structured-agent-session-provider-child' -import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' -import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import type { StructuredAgentSessionStopCause } from './structured-agent-session-adapter' export type { StructuredAgentSessionStopEnding } from './structured-agent-session-host-stop-event' import { recordStopEvent, + STOP_EVENT_DRAIN_TIMEOUT_MS, stopEndsWork, type StructuredAgentSessionStopEnding } from './structured-agent-session-host-stop-event' @@ -50,6 +42,8 @@ export type StructuredAgentSessionLifetimeContext = { publishStatus?: (sessionId: string) => void /** Hands the delivery loop what is queued; for a caller inside the session's serialize. */ wakeDelivery?: (sessionId: string) => void + /** The session's lane, which a close's late proof takes to end the child's record. */ + serialize: (sessionId: string, task: () => Promise) => Promise /** Quit-only snapshot taken immediately before the provider child is stopped. */ restartWitness?: { beforeStop: (sessionId: string) => void @@ -90,43 +84,12 @@ export async function abandonQueuedStructuredAgentSessionMessages( ) } -/** The wind-down this host owes for the session's child. A live child always owes one, whatever a - * previous childless eviction recorded: a remembered tombstone must never outrank the child in - * front of it. */ -function owedProviderChildWindDown( - session: StructuredAgentSessionHostSession -): StructuredAgentSessionProviderChildIdentity | undefined { - return session.child - ? { generation: session.child.generation, fence: session.child.fence } - : session.owesProviderChildWindDown -} - -/** The stop this pass owes. A retry continues the one already asked for, keeping where it was asked; - * any other stop is a new ask, even one with the same cause: a second close closes what came since. */ -function owedStop( - session: StructuredAgentSessionHostSession, - cause: StructuredAgentSessionStopCause, - retry: boolean -): StructuredAgentSessionOwedWindDown | undefined { - const owed = owedProviderChildWindDown(session) - if (!owed) { - return undefined - } - const asked = session.owesProviderChildWindDown - const continues = retry && asked !== undefined && sameProviderChild(asked, owed) - return { - generation: owed.generation, - fence: owed.fence, - cause, - requestedAt: continues ? asked.requestedAt : session.journal.cursor() - } -} - /** - * The agent goes to rest; the conversation stays. Runs the eviction steps under a deadline. A step - * that fails — or runs out of time — aborts the rest and leaves the wind-down owed, so the next - * stop is a real retry. `ending` is how the child's end is told: a user's Stop, the host stopping it - * for a cause (with its text), or an eviction the conversation's close follows. + * The agent goes to rest; the conversation stays. Begins the child's close, or joins the one + * already begun, and waits for the exit's proof as long as a caller may. Still unproven, it throws + * and the close keeps running: a later proof, or the next asker's attempt, ends the record. + * `ending` is how the child's end is told: a user's Stop, the host stopping it for a cause (with + * its text), or an eviction the conversation's close follows. The first stop's ending decides. */ export async function stopStructuredAgentSessionAgentUnderSerialize( context: StructuredAgentSessionLifetimeContext, @@ -134,141 +97,44 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( ending: StructuredAgentSessionStopEnding ): Promise { const session = context.sessions.get(sessionId) - if (!session) { + const child = session?.child + if (!session || !child) { return } - // Judged before the kill: a stop that ends nothing writes nothing. - const recorded = (await stopEndsWork(context, sessionId, session, ending)) - ? recordStopEvent(context, sessionId, session, ending) - : Promise.resolve() - // The obligation OUTLIVES the child. `child` is ended the instant the adapter proves the exit, - // so a step that aborts after that point would otherwise leave the retry reading "no child - // here" and skipping the settlement and the lease release it still owes. - const asked = 'recorded' in ending ? ending.recorded : ending.cause - // A retry finishes the stop that ended the child, so the child's end keeps that stop's cause. - const cause = session.child ? asked : (session.owesProviderChildWindDown?.cause ?? asked) - const owed = owedStop(session, cause, ending.retry === true) - session.owesProviderChildWindDown = owed - const stopping = session.child - const eviction: StructuredAgentSessionEvictionContext = { - sessionId, - // The retry must not re-stop a child the adapter already proved gone, so this stays honest. - hasProviderChild: stopping !== null, - owesProviderChildWindDown: owed !== undefined, - eventSink: context.runtimeState.eventSinkFor(sessionId), - adapter: context.deps.adapter, - logger: context.deps.logger, - ...(context.restartWitness - ? { beforeProviderChildStop: () => context.restartWitness?.beforeStop(sessionId) } - : {}), - // Host state must not disagree with the adapter for the steps in between. - onProviderChildStopped: (verdict) => { - if (stopping) { - endProviderChild(session, { - generation: stopping.generation, - fence: stopping.fence, - cause, - reason: ('reason' in ending ? ending.reason : undefined) ?? null, - duringStartup: stopping.phase === 'starting', - // A later retry that proves the exit still ends the child at the Stop it finishes. - ...(owed ? { endedAt: owed.requestedAt } : {}), - ...verdict - }) - } - context.restartWitness?.stopped(sessionId) - }, - acknowledgeRelease: () => context.deps.adapter.acknowledgeSessionRelease?.(sessionId), - discardSink: () => context.runtimeState.discardEventSink(sessionId), - settleWork: async () => { - // Folded before the fallback's end is built, so the end reads it (`turnEndAfterStop`). - await recorded - const fence = - owed?.fence ?? structuredAgentSessionConversationFence(context.deps.store, sessionId) - const settled = await settleStructuredAgentSessionDeadGeneration({ - journal: session.journal, - sessionId, - fence, - settlementId: `expected-close:${sessionId}:${fence}:${owed?.generation ?? 'unknown'}`, - pendingSubmissionReason: 'provider_closed_before_acknowledgement', - // Only a turn no adapter settled: one with no close, or whose settle threw. Whether it was - // a person's Stop is its event's to say (`turnEndAfterStop`). - verdict: { state: 'interrupted', completedAt: context.now() }, - showUnexpectedExitOutcome: false - }) - if (!settled.ok) { - context.deps.logger.warn("settling a closed agent's work failed", { - scope: 'close-settlement', - sessionId, - error: settled.error - }) - // Without the cause the log names the step and nothing else. - throw new Error('dead generation work settlement failed', { cause: settled.error }) - } - }, - releaseLease: async () => { - if (owed) { - await releaseStoredStructuredAgentSessionOwner({ - store: context.deps.store, - sessionId, - hasProviderChild: true, - expectedFence: owed.fence, - now: context.now() - }) - } - session.owesProviderChildWindDown = undefined - // Whatever ended the child, the row belongs to the conversation: it shows not-running, and - // only the conversation's close forgets it. - context.publishStatus?.(sessionId) - // The stop's own end hands over what waited on it, whichever caller's retry landed. - context.wakeDelivery?.(sessionId) + if (!child.close) { + // Judged before the kill: a stop that ends nothing writes nothing. + const recorded = (await stopEndsWork(context, sessionId, session, ending)) + ? recordStopEvent(context, sessionId, session, ending) + : Promise.resolve() + child.close = { + cause: 'recorded' in ending ? ending.recorded : ending.cause, + reason: ('reason' in ending ? ending.reason : undefined) ?? null, + recorded } + // The event lands before the rows the kill makes the child write, bounded so a slow journal + // never holds the kill. + await withTimeout(recorded, STOP_EVENT_DRAIN_TIMEOUT_MS, undefined) } - try { - await evictStructuredAgentSession( - eviction, - withStructuredAgentSessionEvictionDeadline(STRUCTURED_AGENT_SESSION_EVICTION_STEPS) + if (context.restartWitness) { + await snapshotBeforeStructuredAgentSessionStop( + { + sessionId, + eventSink: context.runtimeState.eventSinkFor(sessionId), + logger: context.deps.logger + }, + () => context.restartWitness?.beforeStop(sessionId) ) - } catch (error) { - if (session.owesProviderChildWindDown === owed && owed) { - session.owesProviderChildWindDown = { ...owed, failedAt: session.journal.cursor() } - } - throw error } -} - -/** - * Retries the wind-down an earlier stop left owed, with that stop's own cause: a child it could - * not prove gone takes no input, so nothing may write to it or start beside it until this lands. - * One pass of the stop, bounded by its own step deadline (10 s): a shorter bound would cut a - * supervised Claude's exit proof (up to about 7 s) short. Resolves whether nothing is owed now; a - * failure is reported, never thrown, and leaves the exit unverifiable, never exited. Landing, the - * stop itself hands over what waited on it. - */ -export async function finishOwedStructuredAgentSessionWindDownUnderSerialize( - context: StructuredAgentSessionLifetimeContext, - sessionId: string -): Promise { - const session = context.sessions.get(sessionId) - const owed = session && pendingProviderChildWindDown(session) - if (!owed) { - return true - } - try { - await stopStructuredAgentSessionAgentUnderSerialize( - context, + if ( + (await joinStructuredAgentSessionChildClose(context, sessionId, child, child.close)) !== + 'exited' + ) { + throw new StructuredAgentSessionEvictionError( + 'stop-provider-child', sessionId, - owed.cause === 'user-stop' - ? { recorded: 'user-stop', retry: true } - : { cause: owed.cause, retry: true } + new Error('provider child exit was not proven') ) - } catch (error) { - context.deps.logger.warn('retrying an unfinished agent stop failed', { - scope: 'owed-stop-retry', - sessionId, - error - }) } - return context.sessions.get(sessionId)?.owesProviderChildWindDown === undefined } /** A close's cause: the user closing this chat, or the host evicting it (quit, idle, teardown). */ @@ -277,13 +143,13 @@ export type StructuredAgentSessionCloseCause = Extract< 'user-close' | 'evict' > -/** Whether the conversation's handle is only a cache now: no child, no wind-down owed, and nothing - * queued or waiting on the provider. */ +/** Whether the conversation's handle is only a cache now: no child, and nothing queued or waiting + * on the provider. */ export function structuredAgentSessionConversationClosable( session: StructuredAgentSessionHostSession ): boolean { return ( - owedProviderChildWindDown(session) === undefined && + session.child === null && !session.journal.submissions().some(isQueuedAgentJournalSubmission) && session.journal.pendingSubmissions().length === 0 ) @@ -312,9 +178,7 @@ export async function closeStructuredAgentSessionConversationUnderSerialize( return true } -/** Stops every provider child owned by this host while keeping failed evictions reachable. A - * session whose child is already stopped but whose wind-down aborted is still in scope — that is - * the retry. */ +/** Stops every provider child owned by this host while keeping failed evictions reachable. */ export async function evictOwnedStructuredAgentSessions( context: StructuredAgentSessionLifetimeContext & { serialize: (sessionId: string, task: () => Promise) => Promise @@ -322,7 +186,7 @@ export async function evictOwnedStructuredAgentSessions( retainOnFailure: Set ): Promise { const ownedSessionIds = [...context.sessions] - .filter(([, session]) => owedProviderChildWindDown(session) !== undefined) + .filter(([, session]) => session.child !== null) .map(([sessionId]) => sessionId) // Retained up front and cleared only once a stop settles: the quit phase is bounded, and a // timeout leaves these still running. Closing their journals underneath them is the one outcome diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts index 757a76a90590..2430884f6a1b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts @@ -11,7 +11,7 @@ import { structuredAgentSessionConversationFence } from './structured-agent-sess /** How a stop ends the child, and why (`lastEndedChild`). A person's Stop wrote its event in its * own step (`recorded` names its reason); any other stop names the reason its event records, with * the host's text for it. Quit writes none: its resume marker's trigger records why. */ -export type StructuredAgentSessionStopEnding = ( +export type StructuredAgentSessionStopEnding = | { recorded: 'user-stop' } | { cause: Exclude @@ -21,19 +21,16 @@ export type StructuredAgentSessionStopEnding = ( * no work its event records. */ resting?: true } -) & { - /** The retry of a stop already owed, set only by that retry: its event, if any, is written. */ - retry?: true -} -/** How long a host stop waits for the session's sink before it judges whether the stop ends work. */ -const STOP_EVENT_DRAIN_TIMEOUT_MS = 1_000 +/** How long a host stop waits for the session's sink before it judges whether the stop ends work, + * and for its event's write before the kill. */ +export const STOP_EVENT_DRAIN_TIMEOUT_MS = 1_000 /** * Whether this stop ends work its event must record: a running turn or an unanswered send, a start's * own included, read once the sink drained what the provider already said. A start that carries - * no send ends nothing. A person's Stop wrote its own event, and quit, the idle sweep's rest and a - * retry of a stop already owed write none. + * no send ends nothing. A person's Stop wrote its own event, and quit and the idle sweep's rest + * write none; a stop that joins a close already begun writes none either. */ export async function stopEndsWork( context: StructuredAgentSessionLifetimeContext, @@ -42,7 +39,7 @@ export async function stopEndsWork( ending: StructuredAgentSessionStopEnding ): Promise { const { child, journal } = session - if ('recorded' in ending || ending.quit || ending.resting || ending.retry || !child) { + if ('recorded' in ending || ending.quit || ending.resting || !child) { return false } // A failed drain has nothing more to deliver, so the journal's read as it stands holds. One @@ -70,8 +67,8 @@ export async function stopEndsWork( ) } -/** Writes this stop's event (`JournalStopEvent`). Issued before the kill and never awaited by it: - * bookkeeping, reported on failure. */ +/** Writes this stop's event (`JournalStopEvent`). Issued before the kill, which waits for it only + * `STOP_EVENT_DRAIN_TIMEOUT_MS`: bookkeeping, reported on failure. */ export function recordStopEvent( context: StructuredAgentSessionLifetimeContext, sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index 6552f8a26016..068a047ffab4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -38,13 +38,16 @@ export type StructuredAgentSessionProviderChildIdentity = { readonly fence: number } -/** A wind-down still owed, with the stop that owes it: a retry finishes that stop. */ -export type StructuredAgentSessionOwedWindDown = StructuredAgentSessionProviderChildIdentity & { +/** The close a stop began for its child. It lives on the child and ends with it: once begun, the + * child takes no input again, and every later stop, start or provider write joins it. */ +export type StructuredAgentSessionChildClose = { + /** The first stop's, which the child's end keeps however many asks join it. */ readonly cause: StructuredAgentSessionStopCause - /** Where the journal stood when the stop was asked for; the child's end is ordered there. */ - readonly requestedAt: AgentJournalCursor - /** Where it stood once the newest pass failed: a message accepted by then waited through a retry. */ - readonly failedAt?: AgentJournalCursor + readonly reason: string | null + /** The Stop event that stop wrote, folded before the work it ends is settled. */ + readonly recorded: Promise + /** The adapter's close while it runs; cleared when it ends unproven, so the next ask retries. */ + attempt?: Promise } /** The provider process behind a conversation. Written only in @@ -56,6 +59,7 @@ export type StructuredAgentSessionProviderChild = StructuredAgentSessionProvider /** The queued message whose delivery started this child, fixed when the start is made; absent * for any other start. In memory only: it tells a restart offer its own start from another. */ readonly startedFor?: string + close?: StructuredAgentSessionChildClose } /** What ending a child established about its provider root. A stop's comes only from @@ -78,8 +82,7 @@ export type StructuredAgentSessionEndedChild = StructuredAgentSessionProviderChi duringStartup: boolean startedFor?: string /** Where the conversation's journal stood when the child ended, to order the end against a - * message's acceptance. A stop's end stands where it was asked for: a message accepted while - * retries proved the exit waited on it, and came after it. */ + * message's acceptance. */ endedAt: AgentJournalCursor } @@ -92,10 +95,6 @@ export type StructuredAgentSessionHostSession = { * has none — so it may not be evicted to free a child, nor have its lease released as an * observed exit. */ child: StructuredAgentSessionProviderChild | null - /** The wind-down this host still owes for a child it started: settling that generation's work - * and handing the lease back. Outlives `child`, which ends the moment the adapter proves the - * exit — an eviction that aborts after that point must still finish it on the next close. */ - owesProviderChildWindDown?: StructuredAgentSessionOwedWindDown lastEndedChild?: StructuredAgentSessionEndedChild } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 44a576e2f781..d98b338abfde 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -158,7 +158,8 @@ export class StructuredAgentSessionHost { ), publishStatus: this.clientDelivery.publishStatusAndSettlement, serialize: (sessionId, task) => this.tasks.trackAttach(this.serialize(sessionId, task)), - now: () => this.now() + now: () => this.now(), + lifetime: () => this.lifetimeContext() }) this.restartResume = createStructuredAgentSessionRestartResume(deps, this.sessions, { ...structuredAgentSessionRestartResumeSurfaces(this, this.now), @@ -193,7 +194,8 @@ export class StructuredAgentSessionHost { sessions: this.sessions, now: () => this.now(), publishStatus: this.clientDelivery.publishStatus, - wakeDelivery: (sessionId: string) => this.conversationDelivery.loop.wake(sessionId) + wakeDelivery: (sessionId: string) => this.conversationDelivery.loop.wake(sessionId), + serialize: this.serialize } satisfies StructuredAgentSessionLifetimeContext } @@ -279,11 +281,8 @@ export class StructuredAgentSessionHost { openConversation: this.conversationDelivery.open, ensureAgent: (sessionId) => agentStart.ensureStructuredAgentSessionAgentForOperation(this.attachContext(), sessionId), - finishOwedStop: (sessionId) => - agentStart.finishOwedStructuredAgentSessionStopForProviderWrite( - this.attachContext(), - sessionId - ), + joinChildClose: (sessionId) => + agentStart.joinClosingStructuredAgentSessionChild(this.attachContext(), sessionId), wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), stopAgent: (sessionId, ending) => this.lifetime.stopAgent(sessionId, ending), wakeQueuedDrain: (sessionId) => this.queued.drain.schedule(sessionId), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts index 7823f65b5355..9cb6dbab4ef2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts @@ -319,7 +319,6 @@ describe('the idle sweep with no child running (P2-22 ii)', () => { providerHoldsDispatch: () => false, stopAgent, stopStartingAgent: stopAgent, - finishOwedWindDown: vi.fn(async () => true), closeConversation, // A failed step fails the test. logger: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts index 96a5f59cf845..1b67f259ed95 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts @@ -15,7 +15,6 @@ import type { AgentJournalRenderItem } from '../../../shared/agent-session-journ import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import { pendingProviderChildWindDown } from './structured-agent-session-provider-child' export const STRUCTURED_AGENT_SESSION_IDLE_SWEEP_INTERVAL_MS = 5 * 60_000 export const STRUCTURED_AGENT_SESSION_IDLE_MS = 30 * 60_000 @@ -37,8 +36,6 @@ export type StructuredAgentSessionIdleSweepDeps = { providerHoldsDispatch: (sessionId: string) => boolean /** Each of these runs inside the session's serialize and never takes it again. */ stopAgent: (sessionId: string) => Promise - /** Retries a stop that did not finish; landing, it hands over what waited on it. */ - finishOwedWindDown: (sessionId: string) => Promise stopStartingAgent: (sessionId: string) => Promise closeConversation: (sessionId: string) => Promise logger: StructuredAgentSessionLogger @@ -111,12 +108,6 @@ export class StructuredAgentSessionIdleSweep { if (!session || this.deps.isDisposed()) { return } - // A stop that did not finish: retry it now, before the idle test, so the rows its settlement - // wrote cannot push the retry out. A running delivery step retries it itself. - if (pendingProviderChildWindDown(session) && !this.deps.deliveryActive(sessionId)) { - await this.deps.finishOwedWindDown(sessionId) - return - } // Owed work is activity, read every tick, so the agent gets a full window once it ends: a child // can read done before the lead's wake-up turn writes anything. if (session.child && session.child.phase !== 'starting' && this.owesWork(sessionId, session)) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index 6a66836fc8a6..68ff515f1272 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -33,9 +33,9 @@ export type StructuredAgentSessionMutationContext = { openConversation: (sessionId: string) => Promise /** Gives the session a provider child; inside the caller's serialize. */ ensureAgent: (sessionId: string) => Promise - /** Finishes a stop an earlier attempt left owed, for an operation that starts no child; inside - * the caller's serialize. */ - finishOwedStop: (sessionId: string) => Promise + /** Joins a close a stop began on the session's child, for an operation that starts no child; + * inside the caller's serialize. */ + joinChildClose: (sessionId: string) => Promise /** A message was accepted: the session's delivery loop hands it over. */ wakeDelivery: (sessionId: string) => void /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts index 5785cfc53e19..2b84d4cfee23 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -5,13 +5,11 @@ // an exit, a failed re-attach, a Stop and an eviction. Each is matched on the child's generation // and fence, so an ending that arrives late for an older child cannot end a newer one. -import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionEndedChild, StructuredAgentSessionHostSession, - StructuredAgentSessionOwedWindDown, StructuredAgentSessionProviderChild, StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' @@ -48,12 +46,9 @@ export function markProviderChildStarted( return child !== null } -/** `endedAt` is a stop's ask; an exit ends where the journal stands. */ export function endProviderChild( session: ChildBearer, - ended: Omit & { - endedAt?: AgentJournalCursor - } + ended: Omit ): boolean { const child = matchingChild(session, ended) if (!child) { @@ -63,7 +58,7 @@ export function endProviderChild( session.lastEndedChild = { ...ended, ...(child.startedFor === undefined ? {} : { startedFor: child.startedFor }), - endedAt: ended.endedAt ?? session.journal.cursor() + endedAt: session.journal.cursor() } return true } @@ -77,15 +72,6 @@ export function failedProviderChildStart( return !session.child && ended?.duringStartup && ended.cause !== 'user-stop' ? ended : null } -/** The owed wind-down an operation reaching the provider finishes first. One owed for another child - * never outranks the child in front of it, which that child's own stop finishes. */ -export function pendingProviderChildWindDown( - session: Pick -): StructuredAgentSessionOwedWindDown | undefined { - const { child, owesProviderChildWindDown: owed } = session - return owed && (!child || sameProviderChild(child, owed)) ? owed : undefined -} - export function sameProviderChild( a: StructuredAgentSessionProviderChildIdentity, b: StructuredAgentSessionProviderChildIdentity diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts index 7c6511dd2f21..f216b6ac8801 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts @@ -84,11 +84,12 @@ export function openForWrite( } /** For an operation the running child performs, which starts none: the conversation, then any - * stop an earlier attempt left owed, so it never reaches a child that takes no input. */ + * close a stop began on that child, which it joins, so it never reaches a child that takes no + * input. */ export function openForProviderWrite( context: Pick< StructuredAgentSessionMutationContext, - 'openConversation' | 'finishOwedStop' | 'deps' + 'openConversation' | 'joinChildClose' | 'deps' >, envelope: AgentSessionMutationEnvelope ): () => Promise { @@ -98,7 +99,7 @@ export function openForProviderWrite( envelope, context.deps.logger ) - return opened.ok ? context.finishOwedStop(envelope.sessionId) : opened + return opened.ok ? context.joinChildClose(envelope.sessionId) : opened } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts index dd0acf82b371..f220d5a0e489 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts @@ -286,9 +286,9 @@ describe('every Stop entry writes its event, with its reason, before it ends the expect(atClose.events?.map((event) => event.reason)).toEqual(['user-stop']) }) - // The idle sweep finishes a stop whose exit was unproven: the same stop, so its event stands alone. + // A later stop joins a close whose exit was unproven: the same close, so its event stands alone. it.each([['user-close' as const], ['evict' as const]])( - 'writes one event for a close (%s) whose exit was unproven, and none for its retry', + 'writes one event for a close (%s) whose exit was unproven, and none for a stop that joins it', async (cause) => { rig = await createQueuedMessageTestRig({ idleSweep: MANUAL_IDLE_SWEEP }) await runningTurn() @@ -296,13 +296,14 @@ describe('every Stop entry writes its event, with its reason, before it ends the const session = () => rig.host.collaboratorsForTests().sessions.get(HOST_TEST_SESSION)! await rig.host.close(HOST_TEST_SESSION, cause).catch(() => undefined) - expect(session().child).not.toBeNull() - expect(session().owesProviderChildWindDown).toMatchObject({ cause }) + expect(session().child?.close).toMatchObject({ cause }) expect(stopEvents()).toEqual([{ reason: cause, turnId: 'turn-1', at: expect.any(Number) }]) - await idleSweep().tick() + await rig.host['tasks'].serialize(HOST_TEST_SESSION, () => + rig.host['lifetime'].stopAgent(HOST_TEST_SESSION, { cause: 'host-stop' }) + ) - expect(session().owesProviderChildWindDown).toBeUndefined() + expect(session().child).toBeNull() expect(stopEvents().map((event) => event.reason)).toEqual([cause]) expect(session().lastEndedChild?.cause).toBe(cause) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index 1f8fc80c863d..2976bf86a0a8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -337,7 +337,7 @@ describe('a chat that closes', () => { expect(closeSession).toHaveBeenCalledOnce() }) - it('settles and releases on the retry when a step after the child stopped aborts', async () => { + it('reports a failed drain after the child stopped, and still settles and releases', async () => { await attach() dispatch.mockResolvedValueOnce({ state: 'admitted' }) const body = hostTestMessage('pending across an aborted eviction') @@ -351,12 +351,9 @@ describe('a chat that closes', () => { failEvictionDrain() const settled = captureSettledSubmissions() - await expect(host.close(SESSION, 'evict')).rejects.toMatchObject({ step: 'drain-published' }) - // The child is proven gone, but the wind-down it owes is not done: nothing settled, no release. - expect(session!.child).toBeNull() - expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') - + // The child is proven gone: the failed drain is bookkeeping, reported, and the rest still runs. await expect(host.close(SESSION, 'evict')).resolves.toBeUndefined() + expect(session!.child).toBeNull() expect(closeSession).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', @@ -645,7 +642,7 @@ describe('an unexpected provider exit', () => { unsubscribe() }) - it('keeps a requested close out of recovery', async () => { + it('ends the record of a requested close as a close, never as a crash to recover from', async () => { await attach() const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -659,7 +656,9 @@ describe('an unexpected provider exit', () => { }) expect(acquire).toHaveBeenCalledOnce() - expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + expect(host['sessions'].get(SESSION)?.child).toBeNull() + expect(host['sessions'].get(SESSION)?.lastEndedChild).not.toHaveProperty('failure') + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') }) it('recovers after a failed lifecycle barrier and dispatches a distinct next message', async () => { @@ -812,19 +811,16 @@ describe('an unexpected provider exit', () => { }) }) -describe('a quit over an eviction that never got its retry', () => { - // Nothing calls `close` a second time when the user quits instead of reopening the chat, so the - // quit sweep is the last thing that can hand the lease back — and it only reaches the session if - // it still counts a stopped child's unfinished wind-down as owed. - it('finishes the wind-down the aborted close left behind', async () => { +describe('a quit after a close whose drain failed', () => { + // The close's own wind-down hands the lease back past a failed drain, so quit finds nothing owed. + it('has nothing of that child left to finish', async () => { await attach() - await sendPending('pending across an abandoned eviction') + await sendPending('pending across a failed drain') const settled = captureSettledSubmissions() failEvictionDrain() - await expect(host.close(SESSION, 'evict')).rejects.toMatchObject({ step: 'drain-published' }) - expect(host['sessions'].get(SESSION)?.child).toBeNull() - expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') + await expect(host.close(SESSION, 'evict')).resolves.toBeUndefined() + expect(host['sessions'].get(SESSION)?.child ?? null).toBeNull() await host.flushAllStreamedEvents() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts index 48500afa12f3..8051dd4fd1c8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts @@ -79,9 +79,9 @@ export type StructuredAgentSessionStopWindDown = { waitsForProvider: boolean; st /** * A session-ending Stop's second step, queued behind its first in the same tick so nothing sent - * meanwhile reaches the child it ends. The Stop has answered: a failure here is reported. The next - * operation that reaches the agent retries the wind-down it leaves owed, and so does the idle - * sweep's next tick. + * meanwhile reaches the child it ends. The Stop has answered: a failure here is reported. A close + * it could not prove keeps the child on record, and the next operation that reaches the agent + * joins that close. */ export async function endStoppedStructuredAgentSession( ctx: Pick, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts index 419f7005c40c..8db92939152e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts @@ -36,6 +36,8 @@ export type StructuredAgentSessionUnexpectedExitContext< flushLifecycle: (sessionId: string) => Promise publishFence: (sessionId: string, session: TSession) => void publishStatus?: (sessionId: string) => void + /** The delivery loop hands over whatever is queued once the child is off the record. */ + wakeDelivery?: (sessionId: string) => void serialize: (sessionId: string, task: () => Promise) => Promise now: () => number logger: StructuredAgentSessionLogger @@ -143,6 +145,7 @@ export async function settleUnexpectedStructuredAgentSessionExit< if (released) { context.publishFence(unexpectedEvent.sessionId, session) } + context.wakeDelivery?.(unexpectedEvent.sessionId) } } }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wind-down-wait-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wind-down-wait-row.ts deleted file mode 100644 index 5f1d03a7ead3..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wind-down-wait-row.ts +++ /dev/null @@ -1,83 +0,0 @@ -// The row a message waiting on an unfinished stop leaves in the chat: why it has not gone out. - -import { agentSessionFailureFact } from '../../../shared/agent-session-failure' -import { - agentSessionFailureWords, - type AgentSessionFailureWordsContext -} from '../../../shared/agent-session-failure-words' -import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' -import { - AGENT_JOURNAL_THREAD_SCOPE, - type AgentJournalItemIdentity -} from '../../../shared/agent-session-journal-types' -import type { - StructuredAgentSessionHostSession, - StructuredAgentSessionProviderChildIdentity -} from './structured-agent-session-host-types' -import { pendingProviderChildWindDown } from './structured-agent-session-provider-child' - -/** Keyed by the child the stop could not prove gone, so every send that waits on it shares a row. */ -export function structuredAgentSessionWindDownWaitIdentity( - owed: StructuredAgentSessionProviderChildIdentity -): AgentJournalItemIdentity { - return { - provider: 'orca', - clientMessageId: `wind-down-wait:${owed.fence}:${owed.generation ?? 'unknown'}` - } -} - -type WaitingSession = Pick< - StructuredAgentSessionHostSession, - 'journal' | 'child' | 'owesProviderChildWindDown' -> - -function windDownWaitRow( - session: WaitingSession, - owed: StructuredAgentSessionProviderChildIdentity -) { - const itemId = agentJournalItemKey(structuredAgentSessionWindDownWaitIdentity(owed)) - return session.journal.snapshot().items.find((item) => item.itemId === itemId) -} - -/** Every queued message already waited through a retry of the owed stop: it was accepted before the - * newest pass failed. Only a new message retries again, and a retry that lands wakes the loop - * itself, so no other commit, each of which wakes the delivery loop, retries. */ -export function structuredAgentSessionWindDownWaitHolds(session: WaitingSession): boolean { - const failedAt = pendingProviderChildWindDown(session)?.failedAt - if (!failedAt || failedAt.epoch !== session.journal.cursor().epoch) { - return false - } - return session.journal - .submissions() - .every( - (submission) => - !isQueuedAgentJournalSubmission(submission) || - (submission.acceptedSequence ?? 0) <= failedAt.sequence - ) -} - -/** Written once per child, so every message that waits on it shares the row. */ -export async function recordStructuredAgentSessionWindDownWait( - session: WaitingSession, - sessionId: string, - input: { - conversationFence: (sessionId: string) => number - failureTextContext: (sessionId: string) => AgentSessionFailureWordsContext - } -): Promise { - const owed = pendingProviderChildWindDown(session) - if (!owed || windDownWaitRow(session, owed)) { - return - } - const identity = structuredAgentSessionWindDownWaitIdentity(owed) - const words = agentSessionFailureWords(agentSessionFailureFact('previousExitUnverifiable'), { - ...input.failureTextContext(sessionId), - surface: 'row' - }) - await session.journal.appendItem( - identity, - { kind: 'status', tone: 'warning', ...words }, - { fence: input.conversationFence(sessionId), turnScope: AGENT_JOURNAL_THREAD_SCOPE } - ) -} From 103598fd637b2916419ab4d455b4f514fea04e76 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:31:23 -0700 Subject: [PATCH 03/22] fix(native-chat): a message refused because the old process's exit is unverifiable says so, and to send again The start failure for a refusal with reason `previousExitUnverifiable` reads "Orca couldn't confirm Claude's previous process ended. Send your message to try again." instead of "Claude couldn't restart." The status-row kind and the refusal reason stay in the shared lists for rows and hosts that still carry them; the catalogs keep one sentence for both. --- src/renderer/src/i18n/en-runtime-required.json | 2 +- src/renderer/src/i18n/locales/en.json | 2 +- src/renderer/src/i18n/locales/es.json | 2 +- src/renderer/src/i18n/locales/fr.json | 2 +- src/renderer/src/i18n/locales/ja.json | 2 +- src/renderer/src/i18n/locales/ko.json | 2 +- src/renderer/src/i18n/locales/zh.json | 2 +- src/shared/agent-session-failure-copy.ts | 3 +-- src/shared/agent-session-failure-words.ts | 7 +++++++ src/shared/agent-session-failure.ts | 2 +- src/shared/agent-session-refusal-details.ts | 4 ++-- 11 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/renderer/src/i18n/en-runtime-required.json b/src/renderer/src/i18n/en-runtime-required.json index 72118c9ab68d..2ce3ec2b2420 100644 --- a/src/renderer/src/i18n/en-runtime-required.json +++ b/src/renderer/src/i18n/en-runtime-required.json @@ -2759,7 +2759,7 @@ "notDelivered": "This message was not delivered.", "notDeliveredSendAgain": "This message was not delivered. Send it again to continue.", "notSignedIn": "{{agent}} is not signed in for the selected account.", - "previousExitUnverifiable": "Orca couldn't confirm {{agent}}'s previous process ended. Messages wait to be sent until Orca confirms it has ended.", + "previousExitUnverifiable": "Orca couldn't confirm {{agent}}'s previous process ended.", "providerExitedRejection": "{{agent}} stopped before this message was sent.", "providerExitedRow": "{{agent}} stopped while this response was in progress. You can continue in this conversation.", "providerRateLimited": "{{agent}} is rate-limited and retrying.", diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 2f29e560803a..9d0a16114491 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -17561,7 +17561,7 @@ "providerRateLimited": "{{agent}} is rate-limited and retrying.", "providerRetrying": "{{agent}} hit a temporary problem and is retrying.", "providerRetryingQuoted": "{{agent}} is retrying: {{detail}}.", - "previousExitUnverifiable": "Orca couldn't confirm {{agent}}'s previous process ended. Messages wait to be sent until Orca confirms it has ended." + "previousExitUnverifiable": "Orca couldn't confirm {{agent}}'s previous process ended." }, "writeNotice": { "notDoneReadHistory": "This chat's history couldn't be loaded.", diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 45afe6f04a07..95d780a4ac47 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -14999,7 +14999,7 @@ "providerRateLimited": "{{agent}} alcanzó un límite de solicitudes y está reintentando.", "providerRetrying": "{{agent}} tuvo un problema temporal y está reintentando.", "providerRetryingQuoted": "{{agent}} está reintentando: {{detail}}.", - "previousExitUnverifiable": "Orca no pudo confirmar que el proceso anterior de {{agent}} terminó. Los mensajes esperan para enviarse hasta que Orca confirme que terminó." + "previousExitUnverifiable": "Orca no pudo confirmar que el proceso anterior de {{agent}} terminó." }, "writeNotice": { "notDoneReadHistory": "No se pudo cargar el historial de este chat.", diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 85a1b64c154a..2d1a996c67db 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -17465,7 +17465,7 @@ "providerRateLimited": "{{agent}} est limité en débit et réessaie.", "providerRetrying": "{{agent}} a rencontré un problème temporaire et réessaie.", "providerRetryingQuoted": "{{agent}} réessaie : {{detail}}.", - "previousExitUnverifiable": "Orca n'a pas pu confirmer que le processus précédent de {{agent}} s'est arrêté. Les messages attendent d'être envoyés jusqu'à ce qu'Orca confirme son arrêt." + "previousExitUnverifiable": "Orca n'a pas pu confirmer que le processus précédent de {{agent}} s'est arrêté." }, "writeNotice": { "notDoneReadHistory": "L'historique de ce chat n'a pas pu être chargé.", diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 0bed9bab4225..a027e5e34e7d 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -17401,7 +17401,7 @@ "providerRateLimited": "{{agent}} はレート制限を受けているため、再試行しています。", "providerRetrying": "{{agent}} で一時的な問題が発生したため、再試行しています。", "providerRetryingQuoted": "{{agent}} は再試行しています: {{detail}}。", - "previousExitUnverifiable": "Orca は {{agent}} の前のプロセスが終了したことを確認できませんでした。Orca が終了を確認するまで、メッセージは送信を待機します。" + "previousExitUnverifiable": "Orca は {{agent}} の前のプロセスが終了したことを確認できませんでした。" }, "writeNotice": { "notDoneReadHistory": "このチャットの履歴を読み込めませんでした。", diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index a503c3347d81..7c01fde23971 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -17401,7 +17401,7 @@ "providerRateLimited": "{{agent}}이(가) 속도 제한에 걸려 다시 시도하고 있습니다.", "providerRetrying": "{{agent}}에 일시적인 문제가 발생하여 다시 시도하고 있습니다.", "providerRetryingQuoted": "{{agent}}이(가) 다시 시도하고 있습니다: {{detail}}.", - "previousExitUnverifiable": "Orca가 {{agent}}의 이전 프로세스가 종료되었는지 확인하지 못했습니다. Orca가 종료를 확인할 때까지 메시지는 전송을 기다립니다." + "previousExitUnverifiable": "Orca가 {{agent}}의 이전 프로세스가 종료되었는지 확인하지 못했습니다." }, "writeNotice": { "notDoneReadHistory": "이 채팅의 기록을 불러오지 못했습니다.", diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 63fc2b87de52..bfb7bd46911c 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -17366,7 +17366,7 @@ "providerRateLimited": "{{agent}} 已被限流，正在重试。", "providerRetrying": "{{agent}} 遇到临时问题，正在重试。", "providerRetryingQuoted": "{{agent}} 正在重试：{{detail}}。", - "previousExitUnverifiable": "Orca 无法确认 {{agent}} 的上一个进程已结束。在 Orca 确认其已结束之前，消息会等待发送。" + "previousExitUnverifiable": "Orca 无法确认 {{agent}} 的上一个进程已结束。" }, "writeNotice": { "notDoneReadHistory": "无法加载此聊天的历史记录。", diff --git a/src/shared/agent-session-failure-copy.ts b/src/shared/agent-session-failure-copy.ts index e6eeab282229..04fa33c4b2bb 100644 --- a/src/shared/agent-session-failure-copy.ts +++ b/src/shared/agent-session-failure-copy.ts @@ -81,8 +81,7 @@ export const AGENT_SESSION_FAILURE_COPY = { providerRateLimited: '{{agent}} is rate-limited and retrying.', providerRetrying: '{{agent}} hit a temporary problem and is retrying.', providerRetryingQuoted: '{{agent}} is retrying: {{detail}}.', - previousExitUnverifiable: - "Orca couldn't confirm {{agent}}'s previous process ended. Messages wait to be sent until Orca confirms it has ended." + previousExitUnverifiable: "Orca couldn't confirm {{agent}}'s previous process ended." } as const export type AgentSessionFailureCopyId = keyof typeof AGENT_SESSION_FAILURE_COPY diff --git a/src/shared/agent-session-failure-words.ts b/src/shared/agent-session-failure-words.ts index f64ba5cb3352..1c9758d81820 100644 --- a/src/shared/agent-session-failure-words.ts +++ b/src/shared/agent-session-failure-words.ts @@ -145,6 +145,13 @@ function couldNot(verb: 'couldNotStart' | 'couldNotRestart'): Sentence { if (fact.refusal?.details?.reason === 'claimConflicted') { return joinSentences([failed, say('terminalAgentHoldsChat'), say('quitTerminalAgent')]) } + // The previous process may still run, so nothing started: that, never that it exited. + if (fact.refusal?.details?.reason === 'previousExitUnverifiable') { + return joinSentences([ + say('previousExitUnverifiable', agent(say, context)), + ...startRetry(say, context) + ]) + } const code = fact.refusal?.code return joinSentences( code && !START_REFUSAL_RESUMABLE[code] diff --git a/src/shared/agent-session-failure.ts b/src/shared/agent-session-failure.ts index b08c5250a150..a3c97850a1ea 100644 --- a/src/shared/agent-session-failure.ts +++ b/src/shared/agent-session-failure.ts @@ -48,7 +48,7 @@ export const AGENT_SESSION_FAILURE_KINDS = [ 'hostStopped', /** The provider is retrying a request its API refused; not a failure yet. */ 'providerRetrying', - /** A message waits on a child a Stop could not prove gone: its exit is unverifiable. */ + /** A child a Stop could not prove gone: its exit is unverifiable. Kept for rows hosts wrote. */ 'previousExitUnverifiable' ] as const export type AgentSessionFailureKind = (typeof AGENT_SESSION_FAILURE_KINDS)[number] diff --git a/src/shared/agent-session-refusal-details.ts b/src/shared/agent-session-refusal-details.ts index 70c6121d03f5..225a0861d46e 100644 --- a/src/shared/agent-session-refusal-details.ts +++ b/src/shared/agent-session-refusal-details.ts @@ -67,8 +67,8 @@ export const AGENT_SESSION_REFUSAL_REASONS = { 'notResumable', 'noProviderChild', 'conversationHeldElsewhere', - /** A Stop could not prove its child gone, and a retry could not either: that child takes no - * input and none starts beside it. Sent with `ownerVerdict: 'unverifiable'`. */ + /** The close a stop began could not prove its child gone within the caller's wait: that child + * takes no input and none starts beside it. Sent with `ownerVerdict: 'unverifiable'`. */ 'previousExitUnverifiable' ], agent_session_conflict: [ From 902354e16468b13b9a451f3a793d52f9f83304d8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 12:35:57 -0700 Subject: [PATCH 04/22] fix(native-chat): a close's verdict is the root's exit alone, and what follows it is logged - A Claude close resolves as soon as the root's exit is proven: the session ends and its `ended` report goes out then. Saving the resume point runs afterwards and a failure is logged, so a slow or hung write never reads as an unproven exit or keeps a dead child on record. - A root that exits after its close came back unproven finishes that close through the same path as any close, so the session's child work is published as ended (background tasks and subagents no longer stay shown running for a dead agent), and a failure there is logged. - Codex logs a refused final row, and reports a root exit whose forced tree kill could not prove the rest of the tree gone the way Claude does, so the host logs it and blocks nothing. - Both adapters take the host's logger for this bookkeeping. --- .../claude-structured-session-adapter.ts | 23 ++-- .../claude-structured-session-close.test.ts | 8 +- .../claude/claude-structured-session-close.ts | 113 +++++++++++------- ...claude-structured-session-recovery.test.ts | 23 ++-- .../claude/claude-structured-session-state.ts | 3 + .../codex-app-server-connection-types.ts | 2 + src/main/codex/codex-app-server-connection.ts | 17 ++- .../codex/codex-structured-session-acquire.ts | 10 +- .../codex/codex-structured-session-adapter.ts | 1 + .../codex/codex-structured-session-close.ts | 20 +++- .../codex/codex-structured-session-state.ts | 3 + .../codex-structured-session-teardown.ts | 24 +++- .../structured-agent-session-runtime.ts | 2 + .../structured-claude-runtime-adapter.ts | 2 + 14 files changed, 161 insertions(+), 90 deletions(-) diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index a752de5165d9..d52177e1d6a6 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -28,8 +28,8 @@ import { } from './claude-structured-session-state' import { closeAllClaudeSessions, - closeClaudePublishedSessionForDeps, - closeClaudeSession + closeClaudeSession, + finishClaudeCloseAfterExit } from './claude-structured-session-close' import { claudeStoppedRequestEndWait } from './claude-request-end-wait' import { @@ -99,14 +99,14 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda emit: (session, _events, event) => this.emit(session, event), handleExit: (sessionId, attempt, error) => observeClaudeSessionExit(this.exitLifecycle, sessionId, attempt, error), - finishClose: (sessionId, attempt) => { - if (this.sessions.get(sessionId)?.connection === attempt.connection) { - // Joins a close still running; finalizes one that came back unproven before the exit. - void closeClaudePublishedSessionForDeps(this.sessions, sessionId, this.deps).catch( - () => undefined - ) - } - }, + finishClose: (sessionId, attempt) => + finishClaudeCloseAfterExit({ + sessions: this.sessions, + sessionId, + connection: attempt.connection, + deps: this.deps, + afterClose: (close) => this.afterClose(sessionId, close) + }), settleExit: (sessionId, exit) => settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit) } @@ -316,7 +316,8 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda sessions: this.sessions, acquisitions: this.acquisitions, ...(this.deps.persistHandle ? { persistHandle: this.deps.persistHandle } : {}), - ...(this.deps.onEvent ? { onEvent: this.deps.onEvent } : {}) + ...(this.deps.onEvent ? { onEvent: this.deps.onEvent } : {}), + ...(this.deps.logger ? { logger: this.deps.logger } : {}) }) closeAll = (): Promise => diff --git a/src/main/claude/claude-structured-session-close.test.ts b/src/main/claude/claude-structured-session-close.test.ts index 0e67b22ade4a..3d61bbb7357f 100644 --- a/src/main/claude/claude-structured-session-close.test.ts +++ b/src/main/claude/claude-structured-session-close.test.ts @@ -126,11 +126,9 @@ describe('Claude published session close lifecycle', () => { ).sessions.get('session-1') const disposeTranslator = vi.spyOn(session!.translator!, 'dispose') - // The exit is proven; the failed cursor write is reported, never read as an unproven exit. - await expect(adapter.closeSession('session-1')).rejects.toMatchObject({ - name: 'AgentSessionAcquisitionExitProvenError', - cause: persistenceError - }) + // The exit is proven; the failed cursor write after it never reads as an unproven exit. + await expect(adapter.closeSession('session-1')).resolves.toBe(true) + await vi.waitFor(() => expect(persistHandle).toHaveBeenCalledOnce()) // The child is provably dead; a failed cursor write may not suppress the end. expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) expect(events.filter((event) => event.type === 'handle')).toHaveLength(0) diff --git a/src/main/claude/claude-structured-session-close.ts b/src/main/claude/claude-structured-session-close.ts index 05d767054094..903f49f052e7 100644 --- a/src/main/claude/claude-structured-session-close.ts +++ b/src/main/claude/claude-structured-session-close.ts @@ -19,6 +19,7 @@ import { closeProcessRegistry } from '../../shared/child-process/close-process-r import { retireClaudeDispatchWaiters } from './claude-structured-dispatch' import { settledClaudeTurnEndLeaf } from './claude-structured-resume-point' import { settleClaudeTurnEndWaiters } from './claude-request-end-wait' +import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' /** The root's own exit was seen first-hand. The lease follows the root, so a descendant * left unverified or seen alive does not hold it. */ @@ -86,6 +87,7 @@ type CloseClaudePublishedSessionInput = { fence: number }) => Promise onEvent?: (event: ClaudeStructuredSessionEvent) => void + logger?: StructuredAgentSessionLogger } async function finalizeClaudePublishedSession( @@ -123,27 +125,10 @@ async function finalizeClaudePublishedSession( } session.childWork.clear() session.backgroundTasks.clear() - const leafUuid = await settledClaudeTurnEndLeaf(session) - const persistence = - session.closePersistence ?? - (session.closePersistence = (async () => { - await input.persistHandle?.({ - sessionId: input.sessionId, - providerSessionId: session.providerSessionId, - leafUuid, - fence: session.fence - }) - })()) - const ended = { - type: 'ended', - sessionId: input.sessionId, - reason: 'claude session closed', - // The host ends the child's record on it, whoever was still waiting on the close. - cause: 'requested-close', - fence: session.fence, - acquisitionGeneration: session.acquisitionGeneration, - observedAt: Date.now() - } as const + // The exit is proven, so the session ends now. Saving its resume point is bookkeeping that + // follows, reported on failure; it never holds the close or reads as an unproven exit. + session.closeFinalized = true + input.sessions.delete(input.sessionId) let callbackError: unknown let callbackThrew = false const deliver = (event: ClaudeStructuredSessionEvent): void => { @@ -154,26 +139,18 @@ async function finalizeClaudePublishedSession( callbackError ??= error } } - let persistenceError: unknown - try { - await persistence - deliver({ - type: 'handle', - sessionId: input.sessionId, - providerSessionId: session.providerSessionId, - leafUuid, - fence: session.fence - }) - } catch (error) { - persistenceError = error - } - // The connection already proved the child dead, so the session has ended whatever the resume - // point's write did: that write is bookkeeping, reported below, never a reason to keep a dead - // child indexed as if its exit were unproven. - session.closeFinalized = true - input.sessions.delete(input.sessionId) if (!session.closeEnded) { session.closeEnded = true + const ended = { + type: 'ended', + sessionId: input.sessionId, + reason: 'claude session closed', + // The host ends the child's record on it, whoever was still waiting on the close. + cause: 'requested-close', + fence: session.fence, + acquisitionGeneration: session.acquisitionGeneration, + observedAt: Date.now() + } as const try { try { session.translator?.handle(ended) @@ -186,18 +163,41 @@ async function finalizeClaudePublishedSession( session.translator?.dispose() } } + session.closePersistence ??= persistClosedClaudeSession(input, session) if (rootExitVerdict) { throw rootExitVerdict } - if (persistenceError !== undefined || callbackThrew) { + if (callbackThrew) { // The exit is proven; only what followed it failed, which the caller reports. - throw new AgentSessionAcquisitionExitProvenError( - persistenceError !== undefined ? persistenceError : callbackError - ) + throw new AgentSessionAcquisitionExitProvenError(callbackError) } return true } +/** The resume point a closed session leaves for the next start, after the close already ended. */ +async function persistClosedClaudeSession( + input: CloseClaudePublishedSessionInput, + session: ClaudeSession +): Promise { + try { + const leafUuid = await settledClaudeTurnEndLeaf(session) + const handle = { + sessionId: input.sessionId, + providerSessionId: session.providerSessionId, + leafUuid, + fence: session.fence + } + await input.persistHandle?.(handle) + input.onEvent?.({ type: 'handle', ...handle }) + } catch (error) { + input.logger?.warn("saving a closed Claude session's resume point failed", { + scope: 'claude-close-resume-point', + sessionId: input.sessionId, + error + }) + } +} + export async function closeClaudePublishedSession( input: CloseClaudePublishedSessionInput ): Promise { @@ -233,11 +233,37 @@ export function closeClaudePublishedSessionForDeps( fence: number }) => Promise onEvent?: (event: ClaudeStructuredSessionEvent) => void + logger?: StructuredAgentSessionLogger } ): Promise { return closeClaudePublishedSession({ sessions, sessionId, ...deps }) } +/** The root exited after a close came back unproven: joins a close still running, or finishes that + * one for this exact child, through `afterClose`, which publishes the session's child work like + * any close. What failed after the exit is reported. */ +export function finishClaudeCloseAfterExit(input: { + sessions: Map + sessionId: string + connection: ClaudeStreamJsonConnection | null + deps: Parameters[2] + afterClose: (close: () => Promise) => Promise +}): void { + const { sessions, sessionId, deps } = input + if (sessions.get(sessionId)?.connection !== input.connection) { + return + } + void input + .afterClose(() => closeClaudePublishedSessionForDeps(sessions, sessionId, deps)) + .catch((error: unknown) => + deps.logger?.warn('finishing a Claude close after its process exited reported', { + scope: 'claude-close-after-exit', + sessionId, + error + }) + ) +} + export async function closeClaudeSession(input: { sessionId: string sessions: Map @@ -249,6 +275,7 @@ export async function closeClaudeSession(input: { fence: number }) => Promise onEvent?: (event: ClaudeStructuredSessionEvent) => void + logger?: StructuredAgentSessionLogger }): Promise { const attempt = input.acquisitions.get(input.sessionId) if (!(await cancelClaudeAcquisitionAttempt(attempt))) { diff --git a/src/main/claude/claude-structured-session-recovery.test.ts b/src/main/claude/claude-structured-session-recovery.test.ts index 7005f93f5837..a91b1d7c1935 100644 --- a/src/main/claude/claude-structured-session-recovery.test.ts +++ b/src/main/claude/claude-structured-session-recovery.test.ts @@ -95,11 +95,11 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { ).sessions.get('session-1') const disposeTranslator = vi.spyOn(session!.translator!, 'dispose') - await expect(adapter.closeSession('session-1')).rejects.toMatchObject({ - name: 'AgentSessionAcquisitionExitProvenError', - cause: callbackError - }) - expect(events.filter((event) => event.type === 'handle')).toHaveLength(1) + // The handle follows the proven close as bookkeeping; its callback's failure is reported. + await expect(adapter.closeSession('session-1')).resolves.toBe(true) + await vi.waitFor(() => + expect(events.filter((event) => event.type === 'handle')).toHaveLength(1) + ) expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) expect(disposeTranslator).toHaveBeenCalledOnce() }) @@ -114,11 +114,9 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { const adapter = adapterFor(claude, {}, [], [], undefined, persistHandle) await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) - // The exit is proven, so the close reports the write and still ends the session. - await expect(adapter.closeSession('session-1')).rejects.toMatchObject({ - name: 'AgentSessionAcquisitionExitProvenError', - cause: persistenceError - }) + // The exit is proven, so the close ends the session; the write after it is bookkeeping. + await expect(adapter.closeSession('session-1')).resolves.toBe(true) + await vi.waitFor(() => expect(persistHandle).toHaveBeenCalledOnce()) await expect(adapter.closeSession('session-1')).resolves.toBe(true) expect(persistHandle).toHaveBeenCalledOnce() }) @@ -147,6 +145,7 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { await adapter.closeSession('session-1') + await vi.waitFor(() => expect(persistedHandles).toHaveLength(1)) expect(persistedHandles).toEqual([ { sessionId: 'session-1', @@ -155,7 +154,9 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { fence: 7 } ]) - expect(events.at(-2)).toEqual({ + // Written after the close already ended the session: its end comes first. + expect(events.at(-2)).toMatchObject({ type: 'ended', cause: 'requested-close' }) + expect(events.at(-1)).toEqual({ type: 'handle', sessionId: 'session-1', providerSessionId: PROVIDER_SESSION_ID, diff --git a/src/main/claude/claude-structured-session-state.ts b/src/main/claude/claude-structured-session-state.ts index 670f28e5531b..876090c07e4d 100644 --- a/src/main/claude/claude-structured-session-state.ts +++ b/src/main/claude/claude-structured-session-state.ts @@ -1,3 +1,4 @@ +import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' import type { AgentJournalDispatchRejection } from '../../shared/agent-session-failure-words' import type { SubmissionRejectionFact } from '../../shared/agent-session-failure' import type { @@ -115,6 +116,8 @@ export type ClaudeStructuredSessionAdapterDeps = { leafUuid: string | null fence: number }) => Promise + /** Where bookkeeping a close or exit does after the child is gone reports a failure. */ + logger?: StructuredAgentSessionLogger /** Advance the durable resume point in place at a turn end; bookkeeping, never a turn failure. */ persistResumePoint?: (input: { sessionId: string diff --git a/src/main/codex/codex-app-server-connection-types.ts b/src/main/codex/codex-app-server-connection-types.ts index 8c6d4c13d9dd..bf33bc5d0b80 100644 --- a/src/main/codex/codex-app-server-connection-types.ts +++ b/src/main/codex/codex-app-server-connection-types.ts @@ -32,4 +32,6 @@ export type CodexAppServerConnection = { resumeReading?: () => void /** Resolves true only after the child emitted `exit` or `close`; false is unproven. */ close: () => Promise + /** The root exited, but the forced kill of its process tree could not prove the tree gone. */ + readonly processTreeUnproven?: boolean } diff --git a/src/main/codex/codex-app-server-connection.ts b/src/main/codex/codex-app-server-connection.ts index e68761040351..070d3e3c301c 100644 --- a/src/main/codex/codex-app-server-connection.ts +++ b/src/main/codex/codex-app-server-connection.ts @@ -79,6 +79,7 @@ export async function openCodexAppServerConnection( let exitObserved = false let closing = false let exitReported = false + let processTreeUnproven = false const exitProof = new RetryableProcessExitProof() /** First terminal cause, or null while the transport is still usable. Set once: * a child that dies reaches us through several listeners, and the specific @@ -258,16 +259,9 @@ export async function openCodexAppServerConnection( if (!exited) { const treeExited = await terminateProcessTree() await waitForProcessExitUntil(exitPromise, FORCED_EXIT_MS) - if (!treeExited && exitObserved) { - // The lease follows the root, which is gone: a child left behind is reported, and - // blocks nothing. - console.warn( - '[codex-app-server] root exited but its process tree was not proven gone', - { - pid: child.pid - } - ) - } + // The lease follows the root, which is gone: a child left behind is reported by the + // owner, and blocks nothing. + processTreeUnproven = !treeExited && exitObserved } } dispatcher.failPending(new Error('codex app-server connection closed')) @@ -282,6 +276,9 @@ export async function openCodexAppServerConnection( get closed() { return closing || exited || terminalError !== null }, + get processTreeUnproven() { + return processTreeUnproven + }, request, notify, respond: (id, result) => writeResponse({ id, result }), diff --git a/src/main/codex/codex-structured-session-acquire.ts b/src/main/codex/codex-structured-session-acquire.ts index bf40afa199bd..5d87745bf8b0 100644 --- a/src/main/codex/codex-structured-session-acquire.ts +++ b/src/main/codex/codex-structured-session-acquire.ts @@ -111,7 +111,14 @@ export async function acquireCodexStructuredSession(input: { previous: previousAttempt }) acquisitions.assertCurrent(sessionId, attempt) - if (!(await closeCodexPublishedSession(sessions, sessionId, deps.onEvent))) { + if ( + !(await closeCodexPublishedSession( + sessions, + sessionId, + deps.onEvent, + deps.logger ? { logger: deps.logger } : {} + )) + ) { throw new Error(`codex app-server for session ${sessionId} could not be stopped`) } acquisitions.assertCurrent(sessionId, attempt) @@ -172,6 +179,7 @@ export async function acquireCodexStructuredSession(input: { error, // The end of a close Orca began, even one that came back unproven before it. ...(exit?.expected ? { closedByOrca: true as const } : {}), + ...(deps.logger ? { logger: deps.logger } : {}), prompts: acquisition.prompts, ...(deps.onEvent ? { onEvent: deps.onEvent } : {}) }) diff --git a/src/main/codex/codex-structured-session-adapter.ts b/src/main/codex/codex-structured-session-adapter.ts index 13a687a72cdc..2d3e1eda8fe1 100644 --- a/src/main/codex/codex-structured-session-adapter.ts +++ b/src/main/codex/codex-structured-session-adapter.ts @@ -76,6 +76,7 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap sessions: this.sessions, acquisitions: this.acquisitions, ...(deps.onEvent ? { onEvent: deps.onEvent } : {}), + ...(deps.logger ? { logger: deps.logger } : {}), forgetNotificationRetries: (sessionId) => this.notificationRetries.clear(sessionId, null) }) } diff --git a/src/main/codex/codex-structured-session-close.ts b/src/main/codex/codex-structured-session-close.ts index 0afd70a70e8c..3ab803101aa0 100644 --- a/src/main/codex/codex-structured-session-close.ts +++ b/src/main/codex/codex-structured-session-close.ts @@ -8,6 +8,7 @@ import { type CodexStructuredSessionEvent } from './codex-structured-session-state' import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' export function handleCodexSessionExit(input: { sessions: Map @@ -18,6 +19,7 @@ export function handleCodexSessionExit(input: { closedByOrca?: true prompts?: CodexSession['prompts'] onEvent?: (event: CodexStructuredSessionEvent) => void + logger?: StructuredAgentSessionLogger }): boolean { const session = input.sessions.get(input.sessionId) if (!session || session.connection !== input.connection || session.ended) { @@ -43,7 +45,14 @@ export function handleCodexSessionExit(input: { // A synchronous sink rejection (usually backpressure) leaves the terminal rows to the host's // exit settlement, which writes its own bounded fallback. The exit itself is observed, so the // session ends either way: holding a dead child as unproven over a refused row would strand it. - session.translator?.handle(event) + const admission = session.translator?.handle(event) ?? { accepted: true } + if (!admission.accepted) { + input.logger?.warn("Codex's final rows were refused; the host settles the turn instead", { + scope: 'codex-exit-rows', + sessionId: input.sessionId, + reason: admission.reason + }) + } session.ended = true // Nothing can echo for this child any more; the journal's pending-submission // recovery is what settles the sends these were armed for. @@ -63,6 +72,7 @@ export async function closeCodexPublishedSession( sessionId: string, onEvent?: (event: CodexStructuredSessionEvent) => void, options?: { + logger?: StructuredAgentSessionLogger requestedClose?: boolean expectedFence?: number expectedAcquisitionGeneration?: string @@ -97,7 +107,8 @@ export async function closeCodexPublishedSession( error: options?.unexpectedReason ?? new Error('codex session closed'), closedByOrca: true, prompts: session.prompts, - ...(onEvent ? { onEvent } : {}) + ...(onEvent ? { onEvent } : {}), + ...(options?.logger ? { logger: options.logger } : {}) }) } sessions.delete(sessionId) @@ -108,7 +119,8 @@ export async function closeCodexSession( sessionId: string, sessions: Map, acquisitions: CodexAcquisitionRegistry, - onEvent?: (event: CodexStructuredSessionEvent) => void + onEvent?: (event: CodexStructuredSessionEvent) => void, + logger?: StructuredAgentSessionLogger ): Promise { const attempt = acquisitions.get(sessionId) if (!(await cancelCodexAcquisitionAttempt(attempt))) { @@ -117,7 +129,7 @@ export async function closeCodexSession( if (attempt) { acquisitions.deleteIfCurrent(sessionId, attempt) } - return closeCodexPublishedSession(sessions, sessionId, onEvent) + return closeCodexPublishedSession(sessions, sessionId, onEvent, logger ? { logger } : {}) } export async function closeAllCodexSessions( diff --git a/src/main/codex/codex-structured-session-state.ts b/src/main/codex/codex-structured-session-state.ts index a47cb5a78fe2..74246b6a7f17 100644 --- a/src/main/codex/codex-structured-session-state.ts +++ b/src/main/codex/codex-structured-session-state.ts @@ -1,3 +1,4 @@ +import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' import type { AgentJournalItemIdentity, AgentSessionJournalIdentity @@ -77,6 +78,8 @@ export type CodexStructuredSessionAdapterDeps = { /** Host capability seam; production uses the native Windows process table. */ isWindowsProcessStartTimeAvailable?: () => boolean onEvent?: (event: CodexStructuredSessionEvent) => void + /** Where bookkeeping a close or exit does after the child is gone reports a failure. */ + logger?: StructuredAgentSessionLogger /** What the session's child work did, delivered after the journal handled the frame. */ onChildWorkEvidence?: (sessionId: string, evidence: AgentChildWorkEvidence[]) => void /** A send admitted earlier: its identity once Codex echoes it, or its rejection when the turn diff --git a/src/main/codex/codex-structured-session-teardown.ts b/src/main/codex/codex-structured-session-teardown.ts index 115e794d8310..e7395fb3c845 100644 --- a/src/main/codex/codex-structured-session-teardown.ts +++ b/src/main/codex/codex-structured-session-teardown.ts @@ -4,6 +4,8 @@ // session owns are cleared exactly once, and only when the child was actually // proven stopped — a refused close leaves the session indexed for a retry. +import { AgentSessionAcquisitionRootExitObservedError } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' import { closeAllCodexSessions, closeCodexPublishedSession, @@ -19,6 +21,7 @@ export type CodexStructuredSessionTeardownDeps = { sessions: Map acquisitions: CodexAcquisitionRegistry onEvent?: (event: CodexStructuredSessionEvent) => void + logger?: StructuredAgentSessionLogger forgetNotificationRetries: (sessionId: string) => void } @@ -26,13 +29,24 @@ export class CodexStructuredSessionTeardown { constructor(private readonly deps: CodexStructuredSessionTeardownDeps) {} close = async (sessionId: string): Promise => { - const closed = await closeCodexSession( + const connection = this.deps.sessions.get(sessionId)?.connection + const closed = this.settled( sessionId, - this.deps.sessions, - this.deps.acquisitions, - this.deps.onEvent + await closeCodexSession( + sessionId, + this.deps.sessions, + this.deps.acquisitions, + this.deps.onEvent, + this.deps.logger + ) ) - return this.settled(sessionId, closed) + if (closed && connection?.processTreeUnproven) { + // The root exited, so the close is proven; its owner reports the children left unconfirmed. + throw new AgentSessionAcquisitionRootExitObservedError( + new Error('codex app-server exited, but its process tree was not proven gone') + ) + } + return closed } forceClose = async (sessionId: string): Promise => { diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index 4c621a5e0c3b..b5efd11efad1 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -270,6 +270,7 @@ async function installOnJournal( host?.publishChildWorkEvidence(sessionId, evidence), onDispatchSettledLate, onPrimaryThreadStoppedRunning: releaseUnansweredDispatches, + logger: deps.logger, onEvent: (event) => { // Every exit, expected or not: the host ends that child's record. if (event.type === 'ended' && 'cause' in event) { @@ -294,6 +295,7 @@ async function installOnJournal( } : {}), onLifecycleEvent: (event) => lifecycle.deliver(event), + logger: deps.logger, onChildWorkEvidence: (sessionId, evidence) => host?.publishChildWorkEvidence(sessionId, evidence), onDispatchSettledLate, diff --git a/src/main/runtime/structured-claude-runtime-adapter.ts b/src/main/runtime/structured-claude-runtime-adapter.ts index 246f5b5eea3f..732515cdde60 100644 --- a/src/main/runtime/structured-claude-runtime-adapter.ts +++ b/src/main/runtime/structured-claude-runtime-adapter.ts @@ -38,6 +38,7 @@ export type StructuredClaudeRuntimeAdapterDeps = { onDispatchSettledLate?: ClaudeStructuredSessionAdapterDeps['onDispatchSettledLate'] onSessionIdle?: ClaudeStructuredSessionAdapterDeps['onSessionIdle'] onChildWorkEvidence?: ClaudeStructuredSessionAdapterDeps['onChildWorkEvidence'] + logger?: ClaudeStructuredSessionAdapterDeps['logger'] } /** The adapter events the host's lifecycle handler consumes, in the host's vocabulary. */ @@ -132,6 +133,7 @@ export function createStructuredClaudeRuntimeAdapter( ...(deps.onDispatchSettledLate ? { onDispatchSettledLate: deps.onDispatchSettledLate } : {}), ...(deps.onSessionIdle ? { onSessionIdle: deps.onSessionIdle } : {}), ...(deps.onChildWorkEvidence ? { onChildWorkEvidence: deps.onChildWorkEvidence } : {}), + ...(deps.logger ? { logger: deps.logger } : {}), ...(deps.openClaudeConnection ? { openConnection: deps.openClaudeConnection } : {}), ...(deps.readProcessStartTime ? { readProcessStartTime: deps.readProcessStartTime } : {}), ...(deps.modelCatalog ? { modelCatalog: deps.modelCatalog } : {}) From 11fb59323f2495a3a9c035b66ac16c7ad2510465 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 12:36:16 -0700 Subject: [PATCH 05/22] fix(native-chat): one handler ends every child's exit, and a join waits on the adapter's own close - One exit handler (`structured-agent-session-child-exit`) ends a child's record for an exit expected or not. `expected` only changes what the chat is told: the stop's cause, its end at the stop's ask, the settlement id, and no crash outcome row. The lease release keeps the exit's evidence; the handoff guard, lifecycle barrier, sink release and adapter acknowledgement apply to both. A Claude journal-sink failure ends in the same step as its stop, as Orca's own fault. - Joining a close is asking the adapter, whose close is memoized while it runs and bounded by its own kill escalation; the host keeps no attempt of its own and no 10 s caller bound. An ask after a close came back unproven runs the stop again. - A close's end is stamped where its stop was asked for (a repeated ask moves it), so the closed chat and failed start checks order a message accepted meanwhile after it. - A start refused because the old exit is unverifiable rejects what was queued in the same step. - The end of a close the host asked for no longer waits on the cross-session recovery chain. - The kill no longer waits for the stop event's write; the journal writes rows in order. --- ...structured-agent-session-attach-context.ts | 4 + .../structured-agent-session-child-close.ts | 200 +++------------- .../structured-agent-session-child-exit.ts | 218 ++++++++++++++++++ ...red-agent-session-claude-root-exit.test.ts | 18 +- ...ssion-claude-stop-exit-ends-record.test.ts | 164 ++++++++++--- ...tured-agent-session-codex-stop-row.test.ts | 21 +- .../structured-agent-session-delivery-loop.ts | 32 ++- ...structured-agent-session-event-recovery.ts | 65 +++--- ...uctured-agent-session-eviction-deadline.ts | 48 ---- .../structured-agent-session-eviction.test.ts | 63 ++--- .../structured-agent-session-eviction.ts | 30 +-- .../structured-agent-session-host-lifetime.ts | 35 +-- ...tructured-agent-session-host-stop-event.ts | 9 +- .../structured-agent-session-host-types.ts | 7 +- .../structured-agent-session-host.ts | 5 +- .../structured-agent-session-lease-release.ts | 47 +--- ...structured-agent-session-provider-child.ts | 8 +- ...agent-session-startup-failure-exit.test.ts | 22 +- ...d-agent-session-stop-event-binding.test.ts | 3 + ...d-agent-session-stop-event-entries.test.ts | 3 + ...ured-agent-session-unexpected-exit.test.ts | 30 +-- ...tructured-agent-session-unexpected-exit.ts | 199 ---------------- ...d-agent-session-lifecycle-delivery.test.ts | 49 ++++ ...ctured-agent-session-lifecycle-delivery.ts | 9 +- 24 files changed, 629 insertions(+), 660 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts create mode 100644 src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts index 90c638c23fb9..38cdb8fdb5be 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts @@ -14,6 +14,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' +import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' import type { StructuredAgentSessionConversationOpenOptions } from './structured-agent-session-conversation-open' @@ -40,6 +41,9 @@ export type StructuredAgentSessionAttachContext = { serialize: (sessionId: string, task: () => Promise) => Promise now: () => number publishStatus: (sessionId: string) => void + /** Joining a stop's close ends the child's record through the one exit handler. */ + endExitedChild: StructuredAgentSessionLifetimeContext['endExitedChild'] + wakeDelivery?: StructuredAgentSessionLifetimeContext['wakeDelivery'] /** The conversation's one open journal, opened when closed; see `conversation-open`. */ openConversation: ( sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index 27bcd9c6fb53..d9232d142124 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -1,57 +1,35 @@ -// A provider child's close: one per child, joined by every stop, start and provider write that -// meets it, and the one place a close's proven exit ends the child's record. +// A provider child's close, which every stop, start and provider write that meets it joins. // // The close lives on the child (`child.close`) and ends with it, so nothing outlives the process -// it is about. A caller waits a bounded time for the exit's proof; past the bound it answers -// `unverifiable` and the close keeps running. Whenever the exit is proven — by a caller still -// waiting, or by the adapter's own report of it — the record ends here. +// it is about. Joining it is asking the adapter to close: the adapter runs one close per child at +// a time and bounds it by its own kill escalation, so every caller waits on the same attempt, and +// a caller that comes after an attempt ended unproven runs it again. The verdict is the root's +// exit alone. Proven, the one exit handler ends the record; a proof that lands with no caller +// waiting reaches that handler as the adapter's report of the exit. -import { withTimeout } from '../../../shared/promise-timeout-fallback' import { refuse } from '../../../shared/agent-session-wire-refusals' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' -import { - evictStructuredAgentSession, - STRUCTURED_AGENT_SESSION_EVICTION_STEPS -} from './structured-agent-session-eviction' -import { - STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS, - withStructuredAgentSessionEvictionDeadline -} from './structured-agent-session-eviction-deadline' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' -import type { - StructuredAgentSessionChildClose, - StructuredAgentSessionHostSession, - StructuredAgentSessionProviderChild -} from './structured-agent-session-host-types' -import { endProviderChild } from './structured-agent-session-provider-child' +import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' -import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' -import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' -import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' +import { releaseStoredStructuredAgentSessionOwnerAfterExit } from './structured-agent-session-lease-release' import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' -/** What a caller learned about the child's exit: proven, or not within its bound. Loss of contact - * and a close that ran out of its own escalation both read `unverifiable`, never `exited`. */ +/** What a caller learned about the child's exit: proven, or not. A close that ran out of its own + * escalation, or a host Orca lost contact with, reads `unverifiable`, never `exited`. */ export type StructuredAgentSessionChildCloseVerdict = 'exited' | 'unverifiable' -/** Joins the child's close, starting another attempt when the last one ended unproven, and waits - * for its proof as long as the caller may. A proof ends the record before this resolves. */ +/** Joins the child's close and, once its root's exit is proven, ends the record. */ export async function joinStructuredAgentSessionChildClose( context: StructuredAgentSessionLifetimeContext, sessionId: string, - child: StructuredAgentSessionProviderChild, - close: StructuredAgentSessionChildClose + child: StructuredAgentSessionProviderChild ): Promise { - close.attempt ??= closeAttempt(context, sessionId, close) - const proven = await withTimeout( - close.attempt, - STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS, - false - ) - if (!proven) { + if (!(await closeProviderRoot(context, sessionId))) { return 'unverifiable' } - await endClosedStructuredAgentSessionChildUnderSerialize(context, sessionId, child) + await context.endExitedChild(sessionId, child, { expected: true, reason: 'closed by Orca' }) + context.restartWitness?.stopped(sessionId) return 'exited' } @@ -75,33 +53,30 @@ export async function joinClosingStructuredAgentSessionChild( if (!child?.close) { return { ok: true } } - return (await joinStructuredAgentSessionChildClose(context, sessionId, child, child.close)) === - 'exited' + return (await joinStructuredAgentSessionChildClose(context, sessionId, child)) === 'exited' ? { ok: true } : { ok: false, refusal: previousExitUnverifiableRefusal() } } -function closeAttempt( +function closeProviderRoot( context: StructuredAgentSessionLifetimeContext, - sessionId: string, - close: StructuredAgentSessionChildClose + sessionId: string ): Promise { const { adapter, logger } = context.deps // An adapter with no close has nothing to stop; anything else must PROVE the exit. const stop = adapter.disposeSession ?? adapter.closeSession - const attempt = ( - stop - ? stopAgentSessionProviderRoot( - () => stop.call(adapter, sessionId), - // The exit is proven; a child process left behind, or a write after it, is reported only. - (error) => - logger.warn("the agent's process exited, but its close did not finish cleanly", { - scope: 'provider-close-after-exit', - sessionId, - error - }) - ) - : Promise.resolve(true) + if (!stop) { + return Promise.resolve(true) + } + return stopAgentSessionProviderRoot( + () => stop.call(adapter, sessionId), + // The exit is proven; a child process left behind, or a write after it, is reported only. + (error) => + logger.warn("the agent's process exited, but its close did not finish cleanly", { + scope: 'provider-close-after-exit', + sessionId, + error + }) ).catch((error: unknown) => { logger.warn("closing the agent's process did not prove it exited", { scope: 'provider-close', @@ -110,14 +85,6 @@ function closeAttempt( }) return false }) - // Unproven, the next ask starts another attempt. A proof that lands after every caller stopped - // waiting needs nothing here: the adapter reports that exit, which ends the record. - void attempt.then((proven) => { - if (!proven && close.attempt === attempt) { - close.attempt = undefined - } - }) - return attempt } /** Whether the lease still names the child this host last proved gone, unreleased: only that @@ -136,7 +103,7 @@ export function structuredAgentSessionEndedChildHoldsLease( ) } -/** Writes the release a proven exit allows when its wind-down could not: a start and the +/** Writes the release a proven exit allows when the exit handler could not: a start and the * handle's close re-derive it, and a failure is reported, never anyone's refusal. Resolves * whether the lease still names that child. */ export async function releaseLeaseOfEndedStructuredAgentSessionChild( @@ -147,12 +114,12 @@ export async function releaseLeaseOfEndedStructuredAgentSessionChild( if (!ended || !structuredAgentSessionEndedChildHoldsLease(context, sessionId)) { return false } - await releaseStoredStructuredAgentSessionOwner({ + await releaseStoredStructuredAgentSessionOwnerAfterExit({ store: context.deps.store, sessionId, - hasProviderChild: true, expectedFence: ended.fence, - now: context.now() + now: context.now(), + ...(ended.reason ? { exitReason: ended.reason } : {}) }).catch((error: unknown) => context.deps.logger.warn("releasing an exited agent's lease failed", { scope: 'ended-child-lease-release', @@ -162,102 +129,3 @@ export async function releaseLeaseOfEndedStructuredAgentSessionChild( ) return structuredAgentSessionEndedChildHoldsLease(context, sessionId) } - -/** The adapter's report that a child it was asked to close is gone ends the record: a proof that - * landed with no caller left waiting, or a root that exited after its close gave up. */ -export function endClosedStructuredAgentSessionChild( - context: StructuredAgentSessionLifetimeContext, - event: StructuredAgentSessionEndedEvent -): Promise { - return context.serialize(event.sessionId, async () => { - const child = context.sessions.get(event.sessionId)?.child - if (child && child.fence === event.fence && child.generation === event.acquisitionGeneration) { - await endClosedStructuredAgentSessionChildUnderSerialize(context, event.sessionId, child) - } - }) -} - -/** One wind-down per child, which every caller that saw its exit proven awaits. */ -const windDowns = new WeakMap>() - -/** - * The child's process is proven gone: it leaves the record, then every wind-down step is - * attempted, each failure reported, none able to keep it on record or gate the next send. Then - * the delivery loop hands over whatever is queued. Joins the wind-down already running for it. - */ -export function endClosedStructuredAgentSessionChildUnderSerialize( - context: StructuredAgentSessionLifetimeContext, - sessionId: string, - child: StructuredAgentSessionProviderChild -): Promise { - const running = windDowns.get(child) - if (running) { - return running - } - const session = context.sessions.get(sessionId) - if (!session || session.child !== child) { - return Promise.resolve() - } - const windDown = windDownClosedChild(context, sessionId, session, child) - windDowns.set(child, windDown) - return windDown -} - -async function windDownClosedChild( - context: StructuredAgentSessionLifetimeContext, - sessionId: string, - session: StructuredAgentSessionHostSession, - child: StructuredAgentSessionProviderChild -): Promise { - const { deps } = context - const close = child.close - endProviderChild(session, { - generation: child.generation, - fence: child.fence, - cause: close?.cause ?? 'evict', - reason: close?.reason ?? null, - duringStartup: child.phase === 'starting', - rootGone: true - }) - context.restartWitness?.stopped(sessionId) - await evictStructuredAgentSession( - { - sessionId, - eventSink: context.runtimeState.eventSinkFor(sessionId), - logger: deps.logger, - acknowledgeRelease: () => deps.adapter.acknowledgeSessionRelease?.(sessionId), - discardSink: () => context.runtimeState.discardEventSink(sessionId), - settleWork: async () => { - // Folded before the fallback's end is built, so the end reads it (`turnEndAfterStop`). - await close?.recorded - const settled = await settleStructuredAgentSessionDeadGeneration({ - journal: session.journal, - sessionId, - fence: child.fence, - settlementId: `expected-close:${sessionId}:${child.fence}:${child.generation ?? 'unknown'}`, - pendingSubmissionReason: 'provider_closed_before_acknowledgement', - // Only a turn no adapter settled: one with no close, or whose settle threw. Whether it - // was a person's Stop is its event's to say (`turnEndAfterStop`). - verdict: { state: 'interrupted', completedAt: context.now() }, - showUnexpectedExitOutcome: false - }) - if (!settled.ok) { - throw new Error('dead generation work settlement failed', { cause: settled.error }) - } - }, - releaseLease: () => - releaseStoredStructuredAgentSessionOwner({ - store: deps.store, - sessionId, - hasProviderChild: true, - expectedFence: child.fence, - now: context.now() - }) - }, - withStructuredAgentSessionEvictionDeadline(STRUCTURED_AGENT_SESSION_EVICTION_STEPS) - ) - // Whatever ended the child, the row belongs to the conversation: it shows not-running, and only - // the conversation's close forgets it. - context.publishStatus?.(sessionId) - context.wakeDelivery?.(sessionId) -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts new file mode 100644 index 000000000000..8a470f209187 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -0,0 +1,218 @@ +// The one handler that ends a provider child's record, whether its exit was expected (a close this +// host asked for) or not (the child died on its own). The exit was seen or proven first-hand, so +// every step after it is bookkeeping: each is attempted and reported, none keeps the child on +// record, and the record ends in `finally`. `expected` changes only what the chat is told. + +import type { SubmissionRejectionFact } from '../../../shared/agent-session-failure' +import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' +import type { + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild +} from './structured-agent-session-host-types' +import { endProviderChild } from './structured-agent-session-provider-child' +import { + releaseStoredStructuredAgentSessionOwnerAfterExit, + type StructuredAgentSessionLeaseStore +} from './structured-agent-session-lease-release' +import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' +import { + captureUnfinishedStructuredAgentSessionWork, + MAX_UNEXPECTED_EXIT_REASON_CHARS, + settleStructuredAgentSessionDeadGeneration, + type DeadGenerationJournal, + unfinishedStructuredAgentSessionWorkWasInterrupted +} from './structured-agent-session-dead-generation-settlement' +import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import { evictStructuredAgentSession } from './structured-agent-session-eviction' +import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' + +/** How the child's root went: the close this host asked for, or a death of its own. */ +export type StructuredAgentSessionChildExit = { + expected: boolean + /** Log text only; the chat's words come from `failure`. */ + reason: string + failure?: SubmissionRejectionFact + /** Host receipt of the exit: the end time of a turn it interrupted. */ + observedAt?: number + startupUnproven?: true +} + +export type StructuredAgentSessionChildExitSession = Pick< + StructuredAgentSessionHostSession, + 'child' | 'lastEndedChild' +> & { journal: DeadGenerationJournal & Pick } + +export type StructuredAgentSessionChildExitContext< + TSession extends StructuredAgentSessionChildExitSession = StructuredAgentSessionHostSession +> = { + store: StructuredAgentSessionLeaseStore + sessions: Map + flushLifecycle: (sessionId: string) => Promise + publishFence: (sessionId: string, session: TSession) => void + publishStatus?: (sessionId: string) => void + /** The delivery loop hands over whatever is queued once the child is off the record. */ + wakeDelivery?: (sessionId: string) => void + serialize: (sessionId: string, task: () => Promise) => Promise + now: () => number + logger: StructuredAgentSessionLogger + /** Lets the child's sink and the adapter's route for it go; absent leaves both to the next attach. */ + route?: { + runtimeState: Pick + acknowledgeRelease: (sessionId: string) => Promise | void + } +} + +/** An adapter's report of a child's exit, for the child still on record with its identity. */ +export function settleStructuredAgentSessionChildExit< + TSession extends StructuredAgentSessionChildExitSession +>( + context: StructuredAgentSessionChildExitContext, + event: StructuredAgentSessionEndedEvent +): Promise { + return context.serialize(event.sessionId, async () => { + const child = context.sessions.get(event.sessionId)?.child + if (!child || child.fence !== event.fence || child.generation !== event.acquisitionGeneration) { + return + } + await endExitedStructuredAgentSessionChildUnderSerialize(context, event.sessionId, child, { + expected: event.cause === 'requested-close', + reason: event.reason, + ...(event.failure ? { failure: event.failure } : {}), + ...(event.observedAt === undefined ? {} : { observedAt: event.observedAt }), + ...(event.startupUnproven ? { startupUnproven: event.startupUnproven } : {}) + }) + }) +} + +/** Ends the record of a child whose exit this host saw or proved; a no-op once it has left. */ +export async function endExitedStructuredAgentSessionChildUnderSerialize< + TSession extends StructuredAgentSessionChildExitSession +>( + context: StructuredAgentSessionChildExitContext, + sessionId: string, + child: StructuredAgentSessionProviderChild, + exit: StructuredAgentSessionChildExit +): Promise { + const session = context.sessions.get(sessionId) + if (!session || session.child !== child) { + return + } + const { expected } = exit + const close = child.close + // Receipt of the exit is the one end time the host may record for a running turn. + const observedAt = exit.observedAt ?? context.now() + // The host's own phase decides, so a provider that omits the flag still gets a start that + // failed told as one: the row says so. + const exitedDuringStartup = exit.startupUnproven === true || child.phase === 'starting' + const endChild = (): void => { + endProviderChild(session, { + generation: child.generation, + fence: child.fence, + // A close keeps the cause of the stop that asked for it, and ends where it was asked. + cause: expected ? (close?.cause ?? 'evict') : 'exit', + reason: expected ? (close?.reason ?? null) : exit.reason, + ...(!expected && exit.failure ? { failure: exit.failure } : {}), + duringStartup: exitedDuringStartup, + // The adapter publishes an exit only once it saw the root go, first-hand or proven. + rootGone: true, + ...(expected && close ? { endedAt: close.requestedAt } : {}) + }) + context.publishStatus?.(sessionId) + } + const record = context.store.getRecord(sessionId) + if (!record || record.lease.handoffStage !== null) { + // An acquisition or recovery already owns this lease's transition. + endChild() + context.wakeDelivery?.(sessionId) + return + } + const unfinishedWork = captureUnfinishedStructuredAgentSessionWork(session.journal) + try { + try { + const barrier = await context.flushLifecycle(sessionId) + if (!barrier.ok) { + logExitFailure(context, sessionId, 'exit-lifecycle-barrier', barrier.error) + } + } catch (error) { + logExitFailure(context, sessionId, 'exit-lifecycle-barrier', error) + } + // Folded before the fallback's end is built, so the end reads it (`turnEndAfterStop`). + await close?.recorded + const generation = child.generation ?? 'unknown' + const settled = await settleStructuredAgentSessionDeadGeneration({ + journal: session.journal, + sessionId, + fence: child.fence, + settlementId: `${expected ? 'expected-close' : 'provider-exit'}:${sessionId}:${child.fence}:${generation}`, + pendingSubmissionReason: expected + ? 'provider_closed_before_acknowledgement' + : 'provider_exited_before_acknowledgement', + // Only a turn no adapter settled. Whether a close ended a person's turn is the Stop event's to + // say (`turnEndAfterStop`); a death of its own interrupted it. + verdict: { state: 'interrupted', completedAt: expected ? context.now() : observedAt }, + failureTextContext: structuredAgentSessionFailureWordsContext(record, session.journal), + // A failed start always says why: no response was running to carry the reason. + showUnexpectedExitOutcome: + !expected && + (exitedDuringStartup || + unfinishedStructuredAgentSessionWorkWasInterrupted( + unfinishedWork, + session.journal, + observedAt + )), + ...(!expected && exit.failure ? { exitFailure: exit.failure } : {}), + ...(!expected && exitedDuringStartup && child.generation + ? { exitedDuringStartup: { generation: child.generation } } + : {}) + }) + if (!settled.ok) { + logExitFailure(context, sessionId, 'exit-settlement', settled.error) + } + } finally { + // The root's exit was observed, so the owner is released even when terminal settlement could + // not be durably accepted. Bare cause: whatever this settlement could not write is settled from + // it later (the settle recording it queues, or the next open or acquire). + let released = false + try { + await releaseStoredStructuredAgentSessionOwnerAfterExit({ + store: context.store, + sessionId, + expectedFence: child.fence, + now: context.now(), + exitObservedAt: observedAt, + exitReason: exit.reason.slice(0, MAX_UNEXPECTED_EXIT_REASON_CHARS) + }) + released = true + } catch (error) { + logExitFailure(context, sessionId, 'exit-owner-release', error) + } + if (context.route) { + const { runtimeState, acknowledgeRelease } = context.route + await evictStructuredAgentSession({ + sessionId, + eventSink: runtimeState.eventSinkFor(sessionId), + logger: context.logger, + discardSink: () => runtimeState.discardEventSink(sessionId), + acknowledgeRelease: () => acknowledgeRelease(sessionId) + }) + } + endChild() + // A reader re-baselines on a death of the child's own; a close Orca asked for moves no fence a + // reader holds, as a client resends a message when its fence moves. + if (released && !expected) { + context.publishFence(sessionId, session) + } + context.wakeDelivery?.(sessionId) + } +} + +function logExitFailure( + context: Pick, + sessionId: string, + scope: string, + error: unknown +): void { + context.logger.warn('settling a provider exit did not finish', { scope, sessionId, error }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index 1867e4f4729f..bf473bfc6df0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -15,6 +15,7 @@ import { } from '../agent-session-journal/journal-host-database-test-support' import type { AgentSessionAttachParams } from './structured-agent-session-attach' import { stopStructuredAgentSessionAgentUnderSerialize } from './structured-agent-session-host-lifetime' +import { endExitedStructuredAgentSessionChildUnderSerialize } from './structured-agent-session-child-exit' import { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' @@ -136,7 +137,22 @@ describe('Claude root-exit stop', () => { sessions, now: () => NOW + 30 * 60_000, publishStatus, - serialize: (_sessionId, task) => task() + endExitedChild: (sessionId, child, exit) => + endExitedStructuredAgentSessionChildUnderSerialize( + { + store, + sessions, + flushLifecycle: (id) => runtimeState.lifecycleBarrier(id), + publishFence: () => undefined, + publishStatus, + serialize: (_sessionId, task) => task(), + now: () => NOW + 30 * 60_000, + logger: deps.logger + }, + sessionId, + child, + exit + ) }, 'session-1', { cause: 'evict' } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index 4e2ca9cf4b15..91ac994d4a74 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -20,7 +20,6 @@ import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-rec import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { StructuredAgentSessionHost } from './structured-agent-session-host' -import { STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS } from './structured-agent-session-eviction-deadline' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' import { HOST_TEST_NOW as NOW, @@ -41,12 +40,14 @@ let store: AgentSessionRecordStore let claude: ReturnType let log: ReturnType let persistHandle: ReturnType Promise>> +let childWork: string[] beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-claude-stop-exit-ends-record-')) resetHostTestOperationIds() log = recordingStructuredAgentSessionLogger() persistHandle = vi.fn(async () => undefined) + childWork = [] claude = fakeClaude({ replayUuid: null, routes: { interrupt: () => ({ still_queued: [], cancelled: [] }) } @@ -71,6 +72,11 @@ beforeEach(async () => { }, onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), persistHandle: () => persistHandle(), + logger: log.logger, + onChildWorkEvidence: (sessionId, evidence) => { + childWork.push(...evidence.map((edge) => edge.type)) + host.publishChildWorkEvidence(sessionId, evidence) + }, openConnection: claude.openConnection, readProcessStartTime: async () => 1_700_000_000_000, now: () => NOW @@ -284,7 +290,7 @@ it('joins the close a Stop could not prove before the next message, then sends i }) }) -it('has a send made while the close still runs join that close, not start another', async () => { +it('has a send made while the close still runs wait for that close, not start another', async () => { const connection = claude.connections[0]! await openTurn(connection) const proof = deferred() @@ -297,58 +303,50 @@ it('has a send made while the close still runs join that close, not start anothe return close() } refuseWritesOnceClosed(connection) - const closeSession = vi.spyOn(adapter, 'closeSession') await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) frame(connection, INTERRUPTED_RESULT) - await send('Carry on.') + // Accepted on the chat's queue behind the Stop's close, which nothing abandons meanwhile. + const sent = send('Carry on.') await eventually(() => expect(attempts).toBe(1)) expect(claude.connections).toHaveLength(1) proof.resolve() + await sent await resumedWith(connection, 'Carry on.') - // One close of the old child, which the send waited on rather than asking for its own. - expect(closeSession).toHaveBeenCalledOnce() + // One close of the old child, which the send waited on rather than starting its own. expect(attempts).toBe(1) expect(connection.closeCount).toBe(1) }) -// The caller's bound is how long it waits, never how long the close may take. -it('ends the record when the proof lands after every caller stopped waiting', async () => { +// The verdict is the root's exit; the resume point written after it is bookkeeping. +it('ends the record at the proven exit though the resume-point write after it never settles', async () => { const connection = claude.connections[0]! - const proof = deferred() - connection.close = async () => { - connection.closed = true - return proof.promise - } - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - let stopped: unknown - try { - const stopping = host['tasks'] - .serialize(SESSION, () => host['lifetime'].stopAgent(SESSION, { cause: 'evict' })) - .then( - () => 'proven', - (error: unknown) => error - ) - await vi.advanceTimersByTimeAsync(STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS + 2_000) - stopped = await stopping - } finally { - vi.useRealTimers() - } - // Unverifiable for that caller, and the close still running: nothing claims the child exited. - expect(stopped).toMatchObject({ step: 'stop-provider-child' }) - expect(child()?.close?.attempt).toBeDefined() - expect(lease()?.claimStatus).toBe('live') - - proof.resolve(true) + await openTurn(connection) + refuseWritesOnceClosed(connection) + persistHandle.mockImplementationOnce(() => new Promise(() => {})) + await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) + frame(connection, INTERRUPTED_RESULT) await eventually(() => expect(child()).toBeNull()) - expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'evict' }) - await eventually(() => expect(lease()?.claimStatus).toBe('released')) + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'user-stop' }) + await send('Carry on.') + await resumedWith(connection, 'Carry on.') }) it('ends the record when the root exits after its close gave up, with no one asking again', async () => { - const connection = await stopWithUnprovenClose(1) + const connection = claude.connections[0]! + // A background task the old agent was running when the Stop's close came back unproven. + frame(connection, { + type: 'system', + subtype: 'task_started', + uuid: 'task-start', + task_id: 'background-1', + task_type: 'local_agent', + is_backgrounded: true + }) + await stopWithUnprovenClose(1) + childWork.length = 0 // The connection reports the root's exit as the end of the close Orca began. connection.handlers.onExit?.(new Error('claude exited'), { expected: true }) @@ -358,6 +356,8 @@ it('ends the record when the root exits after its close gave up, with no one ask expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'user-stop' }) expect(connection.closeCount).toBe(2) expect(claude.connections).toHaveLength(1) + // The child work ends with the session, so nothing is left shown running for a dead agent. + await eventually(() => expect(childWork).toContain('session-ended')) }) it('sends after a proven exit whose resume-point write and lease release both failed', async () => { @@ -377,7 +377,7 @@ it('sends after a proven exit whose resume-point write and lease release both fa // Both reported; neither kept the dead child on record. await eventually(() => expect(scopes()).toEqual( - expect.arrayContaining(['provider-close-after-exit', 'exit-wind-down']) + expect.arrayContaining(['claude-close-resume-point', 'exit-owner-release']) ) ) expect(lease()?.claimStatus).toBe('live') @@ -468,3 +468,93 @@ it('has the idle reaper join a close still unverifiable, ending the record with expect(lease()).toMatchObject({ claimStatus: 'released', ownerProcess: null }) expect(host.hasSession(SESSION)).toBe(false) }) + +it("ends a Claude journal-sink failure as Orca's own fault, never a quiet rest", async () => { + const connection = claude.connections[0]! + await openTurn(connection) + + host['eventRecovery'].recoverAfterSinkFailure(SESSION, new Error('journal write failed')) + + await eventually(() => expect(child()).toBeNull()) + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ + cause: 'exit', + failure: { kind: 'hostFault' } + }) + expect(connection.closeCount).toBe(1) +}) + +it('delivers a message accepted after a tab close was asked for, when the late exit lands first', async () => { + const connection = claude.connections[0]! + closeUnprovenFor(connection, 1) + await expect(host.close(SESSION, 'user-close')).rejects.toThrow() + const closing = child()! + expect(closing.close).toMatchObject({ cause: 'user-close' }) + + // The message is accepted, then the exit report ends the child, before the delivery step runs. + const held = deferred() + const holding = host['tasks'].serialize(SESSION, () => held.promise) + const sent = send('Carry on.') + const ended = host['tasks'].serialize(SESSION, () => + host['eventRecovery'].endExitedChildUnderSerialize(SESSION, closing, { + expected: true, + reason: 'claude session closed' + }) + ) + held.resolve() + await holding + const next = await sent + await ended + + // The close was asked for before the message, so its end closes nothing the message carries. + await resumedWith(connection, 'Carry on.') + expect(await submission(next)).not.toMatchObject({ dispatchState: 'rejected' }) +}) + +it('fails only the message its own refusal is about, never one accepted while that start was refused', async () => { + const connection = claude.connections[0]! + await openTurn(connection) + // The Stop's close and the first send's join both come back unproven; the second's proves. + const close = connection.close + const joined = deferred() + let attempts = 0 + connection.close = async () => { + attempts += 1 + connection.closeCount += 1 + connection.closed = true + if (attempts === 1) { + return false + } + return attempts === 2 ? joined.promise : close() + } + refuseWritesOnceClosed(connection) + await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: true } }) + frame(connection, INTERRUPTED_RESULT) + await laneDrained() + + const first = await send('First.') + await eventually(() => expect(attempts).toBe(2)) + // Accepted while the first send's join still runs, then that join comes back unproven. + const second = send('Second.') + joined.resolve(false) + + const secondId = await second + await eventually(async () => expect((await submission(first))?.dispatchState).toBe('rejected')) + await resumedWith(connection, 'Second.') + expect(await submission(secondId)).not.toMatchObject({ dispatchState: 'rejected' }) +}) + +it('runs the stop again for every ask after a close that came back unproven', async () => { + const connection = await stopWithUnprovenClose(3) + + await send('First.') + await eventually(() => expect(connection.closeCount).toBe(2)) + await laneDrained() + await send('Second.') + await eventually(() => expect(connection.closeCount).toBe(3)) + await laneDrained() + + // Each ask asked again; the third proves the exit. + await send('Third.') + await resumedWith(connection, 'Third.') + expect(connection.closeCount).toBe(4) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts index 6f7ef7ee7d1b..c92b23a49b9b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts @@ -27,7 +27,7 @@ import { hostTestOperationId, resetHostTestOperationIds } from './structured-agent-session-host-test-data' -import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' const CALLER = { callerKey: 'client-1' } @@ -38,6 +38,7 @@ let codex: ReturnType let notify: (method: string, params: unknown) => void /** Read at each start, so a test can say what the next start resumes. */ let launch: { resumeThreadId?: string | null } +let log: ReturnType let disposeSession: MockInstance> beforeEach(async () => { @@ -70,7 +71,7 @@ beforeEach(async () => { }) disposeSession = vi.spyOn(adapter, 'disposeSession') host = new StructuredAgentSessionHost({ - logger: createStructuredAgentSessionLogger(), + logger: (log = recordingStructuredAgentSessionLogger()).logger, store, adapter: Object.assign(adapter, { supportsCreate: () => true }), journalDatabase: openTestJournalHostDatabase(root), @@ -533,4 +534,20 @@ describe('a message after a Codex Stop whose exit was unproven', () => { expect(startedWith(old, 'carry on')).toBe(false) expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ cause: 'user-stop' }) }) + + it('reports a process tree left unproven by a proven root exit, and ends the record anyway', async () => { + await runningTurn() + codex.routes['turn/interrupt'] = () => { + throw interruptFailure('internal error') + } + const old = codex.connections.at(-1)! + // The forced kill saw the root exit but could not prove the rest of the tree gone. + Object.defineProperty(old, 'processTreeUnproven', { get: () => old.closed }) + + await stop() + + await vi.waitFor(() => expect(host['sessions'].get(SESSION)?.child).toBeNull()) + expect(log.entries.map((entry) => entry.fields.scope)).toContain('provider-close-after-exit') + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ rootGone: true }) + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts index 3727040e3c0b..4f25268bc3b0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -115,16 +115,8 @@ export class StructuredAgentSessionDeliveryLoop { return } if (!prepared.ok) { - const { refusal, diagnostic } = prepared - // A conversation no agent ever ran, such as a cleared chat's, failed to start, not restart. - const newSession = this.deps.record(sessionId)?.providerHandleChain.length === 0 - const cause = { - refusal, - ...(diagnostic ? { diagnostic } : {}), - ...(newSession ? { newSession: true as const } : {}) - } await this.deps.serialize(sessionId, () => - this.fail(sessionId, { startKey: null, cause }) + this.fail(sessionId, this.refusedStart(sessionId, prepared)) ) return } @@ -187,6 +179,11 @@ export class StructuredAgentSessionDeliveryLoop { return this.fail(sessionId, failedStart) } const ready = await this.deps.ensureProviderChild(sessionId, oldest.clientMessageId) + if (!ready.ok && ready.refusal.details?.reason === 'previousExitUnverifiable') { + // Failed in the step that was refused: a message accepted, or an exit proven, after it must + // not be failed for a verdict that no longer holds. + return this.fail(sessionId, this.refusedStart(sessionId, ready)) + } if (!ready.ok) { return ready } @@ -254,6 +251,23 @@ export class StructuredAgentSessionDeliveryLoop { return 'continue' } + /** A start the session refused, as the failure every queued message it was for is rejected with. */ + private refusedStart( + sessionId: string, + { refusal, diagnostic }: Extract + ): StartFailure { + // A conversation no agent ever ran, such as a cleared chat's, failed to start, not restart. + const newSession = this.deps.record(sessionId)?.providerHandleChain.length === 0 + return { + startKey: null, + cause: { + refusal, + ...(diagnostic ? { diagnostic } : {}), + ...(newSession ? { newSession: true as const } : {}) + } + } + } + private async fail(sessionId: string, failure: StartFailure): Promise<'stop'> { const session = this.deps.sessions.get(sessionId) if (session) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 1a7742d3e5ff..398d04873d41 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -3,13 +3,18 @@ import type { StructuredAgentSessionLifecycleEvent } from './structured-agent-se import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' import type { StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' import { settleStructuredAgentSessionProviderStarted } from './structured-agent-session-provider-started' -import { settleUnexpectedStructuredAgentSessionExit } from './structured-agent-session-unexpected-exit' -import { endClosedStructuredAgentSessionChild } from './structured-agent-session-child-close' -import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' +import { + endExitedStructuredAgentSessionChildUnderSerialize, + settleStructuredAgentSessionChildExit, + type StructuredAgentSessionChildExit, + type StructuredAgentSessionChildExitContext +} from './structured-agent-session-child-exit' +import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' export class StructuredAgentSessionEventRecovery { private readonly sinkFailures = new Set() @@ -24,19 +29,31 @@ export class StructuredAgentSessionEventRecovery { publishStatus?: (sessionId: string) => void serialize: (sessionId: string, task: () => Promise) => Promise now: () => number - /** Where a close the host asked for ends the child's record. */ - lifetime: () => StructuredAgentSessionLifetimeContext + runtimeState: StructuredAgentSessionHostRuntimeState + wakeDelivery: (sessionId: string) => void } ) {} - private get exitContext() { + private get exitContext(): StructuredAgentSessionChildExitContext { + const { deps, runtimeState } = this.context return { ...this.context, - logger: this.context.deps.logger, - wakeDelivery: this.context.lifetime().wakeDelivery + logger: deps.logger, + route: { + runtimeState, + acknowledgeRelease: (sessionId) => deps.adapter.acknowledgeSessionRelease?.(sessionId) + } } } + /** The one exit handler, for a caller inside the session's serialize that proved the exit. */ + endExitedChildUnderSerialize = ( + sessionId: string, + child: StructuredAgentSessionProviderChild, + exit: StructuredAgentSessionChildExit + ): Promise => + endExitedStructuredAgentSessionChildUnderSerialize(this.exitContext, sessionId, child, exit) + recoverAfterSinkFailure(sessionId: string, error: unknown): void { if (this.sinkFailures.has(sessionId)) { return @@ -47,26 +64,17 @@ export class StructuredAgentSessionEventRecovery { const child = this.context.sessions.get(sessionId)?.child const stop = this.context.deps.adapter.forceCloseSession ?? this.context.deps.adapter.closeSession - if (!child || !stop) { - return null - } - const { fence, generation: acquisitionGeneration } = child - const stopped = await stopAgentSessionProviderRoot(() => stop(sessionId)) - if (!stopped || !acquisitionGeneration) { - return null + if (!child || !stop || !(await stopAgentSessionProviderRoot(() => stop(sessionId)))) { + return } - return { - type: 'ended', - sessionId, + // Ended in the same step as the stop, so no report of that close can end it first as a + // quiet rest: Orca stopped the provider because its own journal failed. + await this.endExitedChildUnderSerialize(sessionId, child, { + expected: false, reason: `journal sink failure: ${error instanceof Error ? error.message : String(error)}`, - // Orca stopped the provider because its own journal failed. - failure: agentSessionFailureFact('hostFault'), - cause: 'unexpected-exit', - fence, - acquisitionGeneration - } as const + failure: agentSessionFailureFact('hostFault') + }) }) - .then((event) => (event ? this.handle(event) : undefined)) .catch((error: unknown) => this.context.deps.logger.warn( 'stopping a provider after its journal failed did not finish', @@ -87,9 +95,6 @@ export class StructuredAgentSessionEventRecovery { if (event.type === 'started') { return settleStructuredAgentSessionProviderStarted(this.context, event) } - if (event.cause === 'requested-close') { - return endClosedStructuredAgentSessionChild(this.context.lifetime(), event) - } - await settleUnexpectedStructuredAgentSessionExit(this.exitContext, event) + await settleStructuredAgentSessionChildExit(this.exitContext, event) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts deleted file mode 100644 index 12b5316ef6f7..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction-deadline.ts +++ /dev/null @@ -1,48 +0,0 @@ -// A bound on how long a caller waits on teardown, never on the teardown itself. -// -// Two uses. A caller joining a child's close waits this long for the exit's proof and then answers -// `unverifiable`; the close keeps running, and a proof that lands later still ends the record. Each -// wind-down step after the exit is bounded the same way, so one hung step is reported and the rest -// still run. - -import type { StructuredAgentSessionEvictionStep } from './structured-agent-session-eviction' - -export const STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS = 10_000 - -export class StructuredAgentSessionEvictionTimeoutError extends Error { - constructor( - readonly step: string, - readonly timeoutMs: number - ) { - super(`agent session eviction step "${step}" did not finish within ${timeoutMs}ms`) - this.name = 'StructuredAgentSessionEvictionTimeoutError' - } -} - -export function withStructuredAgentSessionEvictionDeadline( - steps: readonly StructuredAgentSessionEvictionStep[], - timeoutMs = STRUCTURED_AGENT_SESSION_EVICTION_STEP_TIMEOUT_MS -): readonly StructuredAgentSessionEvictionStep[] { - return steps.map((step) => ({ - name: step.name, - run: async (context) => { - let timer: ReturnType | undefined - try { - await Promise.race([ - Promise.resolve(step.run(context)), - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new StructuredAgentSessionEvictionTimeoutError(step.name, timeoutMs)), - timeoutMs - ) - timer.unref?.() - }) - ]) - } finally { - if (timer) { - clearTimeout(timer) - } - } - } - })) -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts index 269ced39ac59..4cf0ad5ed447 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts @@ -9,7 +9,7 @@ import { StructuredAgentSessionHostRuntimeState } from './structured-agent-sessi import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' -function context(): StructuredAgentSessionEvictionContext & { order: string[] } { +function context(closeError?: Error): StructuredAgentSessionEvictionContext & { order: string[] } { const order: string[] = [] return { order, @@ -19,20 +19,19 @@ function context(): StructuredAgentSessionEvictionContext & { order: string[] } unbind: vi.fn(() => order.push('unbind')), drained: vi.fn(async () => { order.push('drained') - return { ok: true } + return { ok: true as const } }), - close: vi.fn(() => order.push('close')) - } as unknown as StructuredAgentSessionEvictionContext['eventSink'], + close: vi.fn(() => { + order.push('close') + if (closeError) { + throw closeError + } + }) + }, acknowledgeRelease: vi.fn(() => { order.push('acknowledgeRelease') }), - discardSink: vi.fn(() => order.push('discardSink')), - settleWork: vi.fn(async () => { - order.push('settleWork') - }), - releaseLease: vi.fn(async () => { - order.push('releaseLease') - }) + discardSink: vi.fn(() => order.push('discardSink')) } } @@ -45,29 +44,18 @@ function runtimeState(): StructuredAgentSessionHostRuntimeState { } as never) } -describe('the wind-down after a proven exit', () => { - it('drains what the child said before it lets the sink go, then acknowledges the release', async () => { +describe("the route release after a child's exit", () => { + it('lets the sink go before it acknowledges the release', async () => { const ctx = context() await evictStructuredAgentSession(ctx) - expect(ctx.order).toEqual([ - 'drained', - 'settleWork', - 'unbind', - 'close', - 'discardSink', - 'releaseLease', - 'acknowledgeRelease' - ]) + expect(ctx.order).toEqual(['unbind', 'close', 'discardSink', 'acknowledgeRelease']) }) it('names every step, so a failure says which one it was', () => { expect(STRUCTURED_AGENT_SESSION_EVICTION_STEPS.map((step) => step.name)).toEqual([ - 'drain-published', - 'settle-dead-generation', 'stop-publishing', 'close-sink', 'discard-sink', - 'release-lease', 'acknowledge-release' ]) }) @@ -75,30 +63,13 @@ describe('the wind-down after a proven exit', () => { // Bookkeeping for a process already gone: none of it may keep the child on record. it('reports a failed step with its name and still runs every step after it', async () => { const recorded = recordingStructuredAgentSessionLogger() - const ctx = { ...context(), logger: recorded.logger } - ctx.eventSink.drained = vi.fn(async () => { - ctx.order.push('drained') - return { ok: false, error: new Error('append failed') } - }) as unknown as StructuredAgentSessionEvictionContext['eventSink']['drained'] - ctx.releaseLease = vi.fn(async () => { - ctx.order.push('releaseLease') - throw new Error('store unavailable') - }) + const ctx = { ...context(new Error('sink already gone')), logger: recorded.logger } await expect(evictStructuredAgentSession(ctx)).resolves.toBeUndefined() - expect(ctx.order).toEqual([ - 'drained', - 'settleWork', - 'unbind', - 'close', - 'discardSink', - 'releaseLease', - 'acknowledgeRelease' - ]) + expect(ctx.order).toEqual(['unbind', 'close', 'discardSink', 'acknowledgeRelease']) expect(recorded.entries.map((entry) => entry.fields.error)).toEqual([ - expect.objectContaining({ step: 'drain-published' }), - expect.objectContaining({ step: 'release-lease' }) + expect.objectContaining({ step: 'close-sink' }) ]) expect(recorded.entries[0]?.fields.error).toBeInstanceOf(StructuredAgentSessionEvictionError) }) @@ -116,8 +87,6 @@ describe('eviction against the real sink cache', () => { logger: recordingStructuredAgentSessionLogger().logger, eventSink: state.eventSinkFor(sessionId), discardSink: () => state.discardEventSink(sessionId), - settleWork: async () => {}, - releaseLease: async () => {}, acknowledgeRelease: () => {} }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts index e0743e8fb028..2ecd885242c4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts @@ -1,16 +1,12 @@ -// What a host does once its provider child is gone: the bookkeeping that hands the session back. +// What a host does with a provider child's route once its exit has ended the child's record: the +// sink it wrote through and the adapter's index of it. Settling its work and handing the lease back +// are the exit handler's (`structured-agent-session-child-exit`); these come after. // // Teardown is a DATA list, not a method body, for the reason this file exists at all: the host // tracked which sessions were live in a map, and tore them down at three unrelated call sites // (app quit, handoff to a TUI, and error cleanup). Closing a chat was never wired to any of them, // so a provider child outlived the chat that owned it for the whole app session. // -// ORDER. These run only after the child's exit is proven (`structured-agent-session-child-close`). -// Closing it is not silent: the codex adapter emits its `ended` event and flushes coalesced text as -// part of shutting down, and those are the rows that clear the running-turn marker. Draining or -// closing the sink ahead of that drops them, which leaves the durable journal claiming the agent is -// still working. So: drain what the child emitted on its way out, settle, then let the sink go. -// // FAILURE. Each step is bookkeeping for a process already gone, so a step that fails is reported // and the rest still run: none of them may keep a dead child on record or gate the next send. @@ -20,18 +16,13 @@ import type { StructuredAgentSessionLogger } from './structured-agent-session-lo export type StructuredAgentSessionEvictionContext = { sessionId: string - eventSink: DeferredStructuredAgentSessionEventSink + eventSink: Pick logger: StructuredAgentSessionLogger /** Tells the adapter the released lease is done with, so it drops this child's route and index. * The conversation stays: stopping the agent never closes its journal. */ acknowledgeRelease: () => Promise | void /** Drops the cached sink so a later attach mints a fresh one. */ discardSink: () => void - /** Settles work owned by the child after its final callbacks have drained. */ - settleWork: () => Promise - /** Hands the lease back now that this host's child is proven gone. No-ops when the record is - * not this host's to release. */ - releaseLease: () => Promise } /** The resume offer is advisory; a stalled sink must not hold the child's stop behind it. */ @@ -61,16 +52,6 @@ export async function snapshotBeforeStructuredAgentSessionStop( export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSessionEvictionStep[] = [ - { - name: 'drain-published', - run: async (context) => { - const barrier = await context.eventSink.drained() - if (!barrier.ok) { - throw barrier.error - } - } - }, - { name: 'settle-dead-generation', run: (context) => context.settleWork() }, { name: 'stop-publishing', run: (context) => context.eventSink.unbind() }, { name: 'close-sink', run: (context) => context.eventSink.close() }, // Why: the runtime caches one sink per session id and hands the SAME instance to the next @@ -78,9 +59,6 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe // accepts every provider event and publishes none. Attach's own failure path already pairs // these two; eviction has to as well. { name: 'discard-sink', run: (context) => context.discardSink() }, - // The durable lease still names a process this host just proved gone; a start re-derives the - // release from that proof if this write fails. - { name: 'release-lease', run: (context) => context.releaseLease() }, { name: 'acknowledge-release', run: (context) => context.acknowledgeRelease() } ] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 449c425fb69d..0c63f1fe4b48 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -9,7 +9,6 @@ // reentrant, so every public entry point takes it once and calls these. import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' -import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' @@ -21,14 +20,15 @@ import { joinStructuredAgentSessionChildClose } from './structured-agent-session import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' +import type { StructuredAgentSessionChildExit } from './structured-agent-session-child-exit' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import type { StructuredAgentSessionStopCause } from './structured-agent-session-adapter' export type { StructuredAgentSessionStopEnding } from './structured-agent-session-host-stop-event' import { recordStopEvent, - STOP_EVENT_DRAIN_TIMEOUT_MS, stopEndsWork, type StructuredAgentSessionStopEnding } from './structured-agent-session-host-stop-event' @@ -42,8 +42,13 @@ export type StructuredAgentSessionLifetimeContext = { publishStatus?: (sessionId: string) => void /** Hands the delivery loop what is queued; for a caller inside the session's serialize. */ wakeDelivery?: (sessionId: string) => void - /** The session's lane, which a close's late proof takes to end the child's record. */ - serialize: (sessionId: string, task: () => Promise) => Promise + /** The one exit handler (`structured-agent-session-child-exit`), for a caller inside the + * session's serialize that proved its child's exit. */ + endExitedChild: ( + sessionId: string, + child: StructuredAgentSessionProviderChild, + exit: StructuredAgentSessionChildExit + ) => Promise /** Quit-only snapshot taken immediately before the provider child is stopped. */ restartWitness?: { beforeStop: (sessionId: string) => void @@ -101,19 +106,22 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( if (!session || !child) { return } + const cause = 'recorded' in ending ? ending.recorded : ending.cause if (!child.close) { - // Judged before the kill: a stop that ends nothing writes nothing. + // Judged before the kill: a stop that ends nothing writes nothing. Its event is issued before + // the kill and never awaited by it; the journal writes rows in order. const recorded = (await stopEndsWork(context, sessionId, session, ending)) ? recordStopEvent(context, sessionId, session, ending) : Promise.resolve() child.close = { - cause: 'recorded' in ending ? ending.recorded : ending.cause, + cause, reason: ('reason' in ending ? ending.reason : undefined) ?? null, - recorded + recorded, + requestedAt: session.journal.cursor() } - // The event lands before the rows the kill makes the child write, bounded so a slow journal - // never holds the kill. - await withTimeout(recorded, STOP_EVENT_DRAIN_TIMEOUT_MS, undefined) + } else if (child.close.cause === cause) { + // The same stop asked again, such as a second close of the chat, closes what came since. + child.close.requestedAt = session.journal.cursor() } if (context.restartWitness) { await snapshotBeforeStructuredAgentSessionStop( @@ -125,10 +133,7 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( () => context.restartWitness?.beforeStop(sessionId) ) } - if ( - (await joinStructuredAgentSessionChildClose(context, sessionId, child, child.close)) !== - 'exited' - ) { + if ((await joinStructuredAgentSessionChildClose(context, sessionId, child)) !== 'exited') { throw new StructuredAgentSessionEvictionError( 'stop-provider-child', sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts index 2430884f6a1b..923faeb10e04 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts @@ -22,9 +22,8 @@ export type StructuredAgentSessionStopEnding = resting?: true } -/** How long a host stop waits for the session's sink before it judges whether the stop ends work, - * and for its event's write before the kill. */ -export const STOP_EVENT_DRAIN_TIMEOUT_MS = 1_000 +/** How long a host stop waits for the session's sink before it judges whether the stop ends work. */ +const STOP_EVENT_DRAIN_TIMEOUT_MS = 1_000 /** * Whether this stop ends work its event must record: a running turn or an unanswered send, a start's @@ -67,8 +66,8 @@ export async function stopEndsWork( ) } -/** Writes this stop's event (`JournalStopEvent`). Issued before the kill, which waits for it only - * `STOP_EVENT_DRAIN_TIMEOUT_MS`: bookkeeping, reported on failure. */ +/** Writes this stop's event (`JournalStopEvent`). Issued before the kill and never awaited by it: + * bookkeeping, reported on failure. */ export function recordStopEvent( context: StructuredAgentSessionLifetimeContext, sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index 068a047ffab4..ed82fffbe45c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -46,8 +46,9 @@ export type StructuredAgentSessionChildClose = { readonly reason: string | null /** The Stop event that stop wrote, folded before the work it ends is settled. */ readonly recorded: Promise - /** The adapter's close while it runs; cleared when it ends unproven, so the next ask retries. */ - attempt?: Promise + /** Where the journal stood when that stop was asked for: the child's end is ordered there, so a + * message accepted while the exit was being proven came after it. A repeated ask moves it. */ + requestedAt: AgentJournalCursor } /** The provider process behind a conversation. Written only in @@ -82,7 +83,7 @@ export type StructuredAgentSessionEndedChild = StructuredAgentSessionProviderChi duringStartup: boolean startedFor?: string /** Where the conversation's journal stood when the child ended, to order the end against a - * message's acceptance. */ + * message's acceptance. A close's end stands where its stop was asked for. */ endedAt: AgentJournalCursor } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index d98b338abfde..bdb82c8fadb1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -159,7 +159,8 @@ export class StructuredAgentSessionHost { publishStatus: this.clientDelivery.publishStatusAndSettlement, serialize: (sessionId, task) => this.tasks.trackAttach(this.serialize(sessionId, task)), now: () => this.now(), - lifetime: () => this.lifetimeContext() + runtimeState: this.runtimeState, + wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId) }) this.restartResume = createStructuredAgentSessionRestartResume(deps, this.sessions, { ...structuredAgentSessionRestartResumeSurfaces(this, this.now), @@ -195,7 +196,7 @@ export class StructuredAgentSessionHost { now: () => this.now(), publishStatus: this.clientDelivery.publishStatus, wakeDelivery: (sessionId: string) => this.conversationDelivery.loop.wake(sessionId), - serialize: this.serialize + endExitedChild: this.eventRecovery.endExitedChildUnderSerialize } satisfies StructuredAgentSessionLifetimeContext } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts index fe584a50fcee..63e3add288e4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts @@ -1,9 +1,9 @@ -// Handing the durable lease back after eviction stopped this host's child. +// Handing the durable lease back once this host's own provider child is gone. // -// Guarded on `hasProviderChild` for a reason that is not bookkeeping: a session restored only for -// reading, or one a TUI owns, names an owner process this host never started and may still be -// alive. Writing `exit-observed` against that record would release a lease out from under a running -// process and let a second writer in. +// Only an exit this host saw or proved, of the child it started at this fence, may release: a +// session restored only for reading, or one a TUI owns, names an owner process this host never +// started and may still be alive. Writing `exit-observed` against that record would release a lease +// out from under a running process and let a second writer in. import { isSurfaceReleasableAgentSessionRecord, @@ -17,45 +17,16 @@ export type StructuredAgentSessionLeaseStore = Pick< 'getRecord' | 'transitionHandoff' > -export async function releaseStoredStructuredAgentSessionOwner(input: { +/** Releases the lease of the child whose exit this host observed, with that exit's evidence. + * Throws `agent_session_checkpoint_stale` when the record no longer names that child. */ +export async function releaseStoredStructuredAgentSessionOwnerAfterExit(input: { store: StructuredAgentSessionLeaseStore sessionId: string - hasProviderChild: boolean expectedFence: number now: number -}): Promise { - if (!input.hasProviderChild) { - return - } - const record = input.store.getRecord(input.sessionId) - if ( - !record || - record.lease.runtimeFence !== input.expectedFence || - !isSurfaceReleasableAgentSessionRecord(record) - ) { - return - } - await releaseStoredAgentSessionOwnerAfterSurfaceClose(input.store, { - sessionId: input.sessionId, - expectedFence: input.expectedFence, - now: input.now - }) -} - -/** Releases only the exact provider child whose exit the adapter positively observed. */ -export async function releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit(input: { - store: StructuredAgentSessionLeaseStore - sessionId: string - expectedFence: number - expectedAcquisitionGeneration: string - acquisitionGeneration: string | null - now: number exitObservedAt?: number exitReason?: string }): Promise { - if (input.acquisitionGeneration !== input.expectedAcquisitionGeneration) { - throw new Error('agent_session_checkpoint_stale') - } const record = input.store.getRecord(input.sessionId) if ( !record || @@ -68,7 +39,7 @@ export async function releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExi sessionId: input.sessionId, expectedFence: input.expectedFence, now: input.now, - exitObservedAt: input.exitObservedAt, + ...(input.exitObservedAt === undefined ? {} : { exitObservedAt: input.exitObservedAt }), ...(input.exitReason ? { exitReason: input.exitReason } : {}) }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts index 2b84d4cfee23..d4236f395da4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -5,6 +5,7 @@ // an exit, a failed re-attach, a Stop and an eviction. Each is matched on the child's generation // and fence, so an ending that arrives late for an older child cannot end a newer one. +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { @@ -46,9 +47,12 @@ export function markProviderChildStarted( return child !== null } +/** `endedAt` is a close's ask; an exit of the child's own ends where the journal stands. */ export function endProviderChild( session: ChildBearer, - ended: Omit + ended: Omit & { + endedAt?: AgentJournalCursor + } ): boolean { const child = matchingChild(session, ended) if (!child) { @@ -58,7 +62,7 @@ export function endProviderChild( session.lastEndedChild = { ...ended, ...(child.startedFor === undefined ? {} : { startedFor: child.startedFor }), - endedAt: session.journal.cursor() + endedAt: ended.endedAt ?? session.journal.cursor() } return true } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index 8235d0d67054..07f54bc2ab4a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -6,10 +6,10 @@ import { } from '../../../shared/agent-session-record.test-fixture' import { structuredAgentSessionCompactBody } from './structured-agent-session-command-turn' import { - settleUnexpectedStructuredAgentSessionExit, - type StructuredAgentSessionUnexpectedExitContext, - type StructuredAgentSessionUnexpectedExitSession -} from './structured-agent-session-unexpected-exit' + settleStructuredAgentSessionChildExit, + type StructuredAgentSessionChildExitContext, + type StructuredAgentSessionChildExitSession +} from './structured-agent-session-child-exit' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' const SESSION = 'session-1' @@ -17,7 +17,7 @@ const GENERATION = 'generation-1' const REASON = 'claude stream-json exited (code 1): session limit reached' const STARTUP_TEXT = 'Claude stopped before it finished starting. Send your message to try again.' -function startedSession(): StructuredAgentSessionUnexpectedExitSession & { +function startedSession(): StructuredAgentSessionChildExitSession & { journal: { appendLifecycleBatch: ReturnType } } { return { @@ -34,7 +34,7 @@ function startedSession(): StructuredAgentSessionUnexpectedExitSession & { } } -function contextFor(session: StructuredAgentSessionUnexpectedExitSession) { +function contextFor(session: StructuredAgentSessionChildExitSession) { let record: AgentSessionRecord = agentSessionRecordFixture( agentSessionLeaseFixture({ sessionId: SESSION, @@ -47,7 +47,7 @@ function contextFor(session: StructuredAgentSessionUnexpectedExitSession) { unreconciled: false }) ) - const context: StructuredAgentSessionUnexpectedExitContext = { + const context: StructuredAgentSessionChildExitContext = { logger: recordingStructuredAgentSessionLogger().logger, store: { getRecord: () => record, @@ -78,7 +78,7 @@ describe('a provider that ends before it finished starting', () => { it('tells the user why, even with no response in progress', async () => { const session = startedSession() - await settleUnexpectedStructuredAgentSessionExit(contextFor(session), { + await settleStructuredAgentSessionChildExit(contextFor(session), { ...ended, // The adapter typed the start's own failure; the host keeps it rather than reword it. failure: { kind: 'notSignedIn' }, @@ -106,7 +106,7 @@ describe('a provider that ends before it finished starting', () => { it('keeps an ordinary idle exit silent', async () => { const session = startedSession() - await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) + await settleStructuredAgentSessionChildExit(contextFor(session), ended) expect(session.child).toBeNull() expect(session.journal.appendLifecycleBatch).not.toHaveBeenCalled() @@ -118,7 +118,7 @@ describe('a provider that ends before it finished starting', () => { child: { generation: GENERATION, fence: 7, phase: 'starting' as const } } - await settleUnexpectedStructuredAgentSessionExit(contextFor(session), { + await settleStructuredAgentSessionChildExit(contextFor(session), { ...ended, failure: { kind: 'providerExited', detail: { text: REASON, audience: 'log' } } }) @@ -157,7 +157,7 @@ describe('a provider that ends before it finished starting', () => { } } - await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) + await settleStructuredAgentSessionChildExit(contextFor(session), ended) const text = 'Claude stopped before it finished starting. Run /compact again.' expect(session.journal.rejectPendingSubmissions).toHaveBeenCalledWith( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts index 2654b20ac730..fb36ab2d5e09 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts @@ -118,6 +118,9 @@ async function mailTurn(): Promise { async function evictedAt(): Promise { let atClose: JournalStopEvent[] = [] rig.closeSession.mockImplementationOnce(async () => { + // The event is issued before the kill, never awaited by it: the journal writes it ahead of + // anything the kill makes the child write. + await new Promise((resolve) => setImmediate(resolve)) atClose = stopEvents() return true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts index f220d5a0e489..1d7781c98869 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts @@ -45,6 +45,9 @@ function stopEvents(): JournalStopEvent[] { function stopEventsAtClose(): { events: JournalStopEvent[] | null } { const seen: { events: JournalStopEvent[] | null } = { events: null } rig.closeSession.mockImplementationOnce(async () => { + // The event is issued before the kill, never awaited by it: the journal writes it ahead of + // anything the kill makes the child write. + await new Promise((resolve) => setImmediate(resolve)) seen.events = stopEvents() return true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 4f4e414657de..e5222eae1b6a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -9,10 +9,10 @@ import { import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' import { - settleUnexpectedStructuredAgentSessionExit, - type StructuredAgentSessionUnexpectedExitContext, - type StructuredAgentSessionUnexpectedExitSession -} from './structured-agent-session-unexpected-exit' + settleStructuredAgentSessionChildExit, + type StructuredAgentSessionChildExitContext, + type StructuredAgentSessionChildExitSession +} from './structured-agent-session-child-exit' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' const exitOutcome = (agent: string): string => @@ -105,7 +105,7 @@ describe('provider-exit settlement', () => { } } as unknown as StructuredAgentSessionHostSession - await settleUnexpectedStructuredAgentSessionExit( + await settleStructuredAgentSessionChildExit( // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit settlement reads only these members of its context; the store double is partial. { logger: recordingStructuredAgentSessionLogger().logger, @@ -190,7 +190,7 @@ describe('provider-exit settlement', () => { transitionHandoff: async () => ({ lease: { runtimeFence: 8 } }) } - await settleUnexpectedStructuredAgentSessionExit( + await settleStructuredAgentSessionChildExit( { store, sessions: new Map([[SESSION, session]]), @@ -285,7 +285,7 @@ describe('provider-exit settlement', () => { epoch: 'epoch-1', sequence: 3 })) - const session: StructuredAgentSessionUnexpectedExitSession = { + const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), @@ -298,7 +298,7 @@ describe('provider-exit settlement', () => { } const { store } = mutableStore() - const context: StructuredAgentSessionUnexpectedExitContext = { + const context: StructuredAgentSessionChildExitContext = { logger: recordingStructuredAgentSessionLogger().logger, store, sessions: new Map([[SESSION, session]]), @@ -316,7 +316,7 @@ describe('provider-exit settlement', () => { serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1_234 } - await settleUnexpectedStructuredAgentSessionExit(context, { + await settleStructuredAgentSessionChildExit(context, { type: 'ended', sessionId: SESSION, reason: 'provider exited after completing the turn', @@ -344,7 +344,7 @@ describe('provider-exit settlement', () => { it('settles a submission the dead child never acknowledged', async () => { const markPendingSubmissionsUnknown = vi.fn(async () => ['client-1']) - const session: StructuredAgentSessionUnexpectedExitSession = { + const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), @@ -358,7 +358,7 @@ describe('provider-exit settlement', () => { } const { store } = mutableStore() - const context: StructuredAgentSessionUnexpectedExitContext = { + const context: StructuredAgentSessionChildExitContext = { logger: recordingStructuredAgentSessionLogger().logger, store, sessions: new Map([[SESSION, session]]), @@ -367,7 +367,7 @@ describe('provider-exit settlement', () => { serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1 } - await settleUnexpectedStructuredAgentSessionExit(context, { + await settleStructuredAgentSessionChildExit(context, { type: 'ended', sessionId: SESSION, reason: 'provider exited', @@ -397,7 +397,7 @@ describe('provider-exit settlement', () => { }) it('releases without offering a restart while terminal settlement is failing', async () => { - const session: StructuredAgentSessionUnexpectedExitSession = { + const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), @@ -423,7 +423,7 @@ describe('provider-exit settlement', () => { acquisitionGeneration: GENERATION } const { store } = mutableStore() - const context: StructuredAgentSessionUnexpectedExitContext = { + const context: StructuredAgentSessionChildExitContext = { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: false, error: new Error('sink failed') }), @@ -432,7 +432,7 @@ describe('provider-exit settlement', () => { now: () => 1, logger: log.logger } - await settleUnexpectedStructuredAgentSessionExit(context, event) + await settleStructuredAgentSessionChildExit(context, event) expect(session.child).toBeNull() expect(publishFence).toHaveBeenCalledTimes(1) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts deleted file mode 100644 index 8db92939152e..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts +++ /dev/null @@ -1,199 +0,0 @@ -import type { AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words' -import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' -import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import { endProviderChild } from './structured-agent-session-provider-child' -import { - releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit, - type StructuredAgentSessionLeaseStore -} from './structured-agent-session-lease-release' -import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' -import { - captureUnfinishedStructuredAgentSessionWork, - MAX_UNEXPECTED_EXIT_REASON_CHARS, - settleStructuredAgentSessionDeadGeneration, - type DeadGenerationJournal, - unfinishedStructuredAgentSessionWorkWasInterrupted -} from './structured-agent-session-dead-generation-settlement' -import type { StructuredAgentSessionTurnVerdict } from './structured-agent-session-stale-turn-verdict' -import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' - -type UnexpectedExitLifecycleEvent = StructuredAgentSessionEndedEvent & { - cause: 'unexpected-exit' -} - -export type StructuredAgentSessionUnexpectedExitSession = Pick< - StructuredAgentSessionHostSession, - 'child' | 'lastEndedChild' -> & { journal: DeadGenerationJournal & Pick } - -export type StructuredAgentSessionUnexpectedExitContext< - TSession extends StructuredAgentSessionUnexpectedExitSession = StructuredAgentSessionHostSession -> = { - store: StructuredAgentSessionLeaseStore - sessions: Map - flushLifecycle: (sessionId: string) => Promise - publishFence: (sessionId: string, session: TSession) => void - publishStatus?: (sessionId: string) => void - /** The delivery loop hands over whatever is queued once the child is off the record. */ - wakeDelivery?: (sessionId: string) => void - serialize: (sessionId: string, task: () => Promise) => Promise - now: () => number - logger: StructuredAgentSessionLogger -} - -export async function settleUnexpectedStructuredAgentSessionExit< - TSession extends StructuredAgentSessionUnexpectedExitSession ->( - context: StructuredAgentSessionUnexpectedExitContext, - event: StructuredAgentSessionEndedEvent -): Promise { - if (event.cause !== 'unexpected-exit') { - return - } - const unexpectedEvent = event as UnexpectedExitLifecycleEvent - // Receipt of the exit is the one end time the host may record for a running turn. - const observedAt = event.observedAt ?? context.now() - return context.serialize(unexpectedEvent.sessionId, async () => { - const session = context.sessions.get(unexpectedEvent.sessionId) - const child = session?.child - if ( - !session || - !child || - child.fence !== unexpectedEvent.fence || - child.generation !== unexpectedEvent.acquisitionGeneration - ) { - return - } - // The host's own phase decides, so a provider that omits the flag still gets a start that - // failed told as one: the row says so. - const exitedDuringStartup = - unexpectedEvent.startupUnproven === true || child.phase === 'starting' - const endChild = (): void => { - endProviderChild(session, { - generation: child.generation, - fence: child.fence, - cause: 'exit', - reason: unexpectedEvent.reason, - ...(unexpectedEvent.failure ? { failure: unexpectedEvent.failure } : {}), - duringStartup: exitedDuringStartup, - // The adapter publishes an exit only once it saw the root go, first-hand or proven. - rootGone: true - }) - context.publishStatus?.(unexpectedEvent.sessionId) - } - const record = context.store.getRecord(unexpectedEvent.sessionId) - if (!record || record.lease.handoffStage !== null) { - // An acquisition or recovery already owns this lease's transition. - endChild() - return - } - - const stableSettlementId = providerExitSettlementId(unexpectedEvent) - const unfinishedWork = captureUnfinishedStructuredAgentSessionWork(session.journal) - try { - try { - const barrier = await context.flushLifecycle(unexpectedEvent.sessionId) - if (!barrier.ok) { - logExitFailure(context, unexpectedEvent, 'exit-lifecycle-barrier', barrier.error) - } - } catch (error) { - logExitFailure(context, unexpectedEvent, 'exit-lifecycle-barrier', error) - } - await retryUnexpectedExitSettlement({ - context, - event: unexpectedEvent, - journal: session.journal, - fence: child.fence, - stableSettlementId, - verdict: { state: 'interrupted', completedAt: observedAt }, - exitedDuringStartup, - failureTextContext: structuredAgentSessionFailureWordsContext(record, session.journal), - // A failed start always says why: no response was running to carry the reason. - showUnexpectedExitOutcome: - exitedDuringStartup || - unfinishedStructuredAgentSessionWorkWasInterrupted( - unfinishedWork, - session.journal, - observedAt - ) - }) - } finally { - // Provider exit was positively observed, so release the owner even when - // terminal settlement could not be durably accepted. - let released: Awaited< - ReturnType - > | null = null - try { - released = await releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit({ - store: context.store, - sessionId: unexpectedEvent.sessionId, - expectedFence: unexpectedEvent.fence, - expectedAcquisitionGeneration: unexpectedEvent.acquisitionGeneration, - acquisitionGeneration: child.generation, - now: context.now(), - exitObservedAt: observedAt, - // Bare cause: whatever this settlement could not write is settled from it later (the - // settle recording it queues, or the next open or acquire); `exit-observed` says the rest. - exitReason: unexpectedEvent.reason.slice(0, MAX_UNEXPECTED_EXIT_REASON_CHARS) - }) - } catch (error) { - logExitFailure(context, unexpectedEvent, 'exit-owner-release', error) - } finally { - endChild() - if (released) { - context.publishFence(unexpectedEvent.sessionId, session) - } - context.wakeDelivery?.(unexpectedEvent.sessionId) - } - } - }) -} - -async function retryUnexpectedExitSettlement(input: { - context: Pick - event: UnexpectedExitLifecycleEvent - journal: DeadGenerationJournal - fence: number - stableSettlementId: string - verdict: StructuredAgentSessionTurnVerdict - exitedDuringStartup: boolean - failureTextContext: AgentSessionFailureWordsContext - showUnexpectedExitOutcome?: boolean -}): Promise { - const settled = await settleStructuredAgentSessionDeadGeneration({ - journal: input.journal, - sessionId: input.event.sessionId, - fence: input.fence, - settlementId: input.stableSettlementId, - verdict: input.verdict, - pendingSubmissionReason: 'provider_exited_before_acknowledgement', - showUnexpectedExitOutcome: input.showUnexpectedExitOutcome, - ...(input.event.failure ? { exitFailure: input.event.failure } : {}), - failureTextContext: input.failureTextContext, - ...(input.exitedDuringStartup - ? { exitedDuringStartup: { generation: input.event.acquisitionGeneration } } - : {}) - }) - if (!settled.ok) { - logExitFailure(input.context, input.event, 'exit-settlement', settled.error) - } -} - -function logExitFailure( - context: Pick, - event: UnexpectedExitLifecycleEvent, - scope: string, - error: unknown -): void { - context.logger.warn('settling a provider exit did not finish', { - scope, - sessionId: event.sessionId, - error - }) -} - -function providerExitSettlementId(event: UnexpectedExitLifecycleEvent): string { - return `provider-exit:${event.sessionId}:${event.fence}:${event.acquisitionGeneration}` -} diff --git a/src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts b/src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts new file mode 100644 index 000000000000..b236c81d594b --- /dev/null +++ b/src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import { recordingStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' +import { createStructuredAgentSessionLifecycleDelivery } from './structured-agent-session-lifecycle-delivery' + +function ended( + sessionId: string, + cause: 'unexpected-exit' | 'requested-close' +): StructuredAgentSessionLifecycleEvent { + return { + type: 'ended', + sessionId, + reason: 'exited', + cause, + fence: 1, + acquisitionGeneration: 'generation-1' + } +} + +describe('provider exits reaching the host', () => { + // A close the host asked for ends on its own session's lane; another chat's crash recovery, + // which can run a whole reacquisition, must not hold it up. + it('ends a requested close without waiting behind another chat in recovery', async () => { + const handled: string[] = [] + let releaseRecovery = (): void => {} + const recovery = new Promise((resolve) => { + releaseRecovery = resolve + }) + const delivery = createStructuredAgentSessionLifecycleDelivery({ + handle: async (event) => { + if (event.type === 'ended' && event.cause === 'unexpected-exit') { + await recovery + } + handled.push(event.sessionId) + }, + logger: recordingStructuredAgentSessionLogger().logger, + drainObservedExits: async () => {} + }) + + delivery.deliver(ended('crashed', 'unexpected-exit')) + delivery.deliver(ended('closed', 'requested-close')) + await new Promise((resolve) => setImmediate(resolve)) + + expect(handled).toEqual(['closed']) + releaseRecovery() + await delivery.drain() + expect(handled).toEqual(['closed', 'crashed']) + }) +}) diff --git a/src/main/runtime/structured-agent-session-lifecycle-delivery.ts b/src/main/runtime/structured-agent-session-lifecycle-delivery.ts index ccf026de359c..ba307c2cfb0a 100644 --- a/src/main/runtime/structured-agent-session-lifecycle-delivery.ts +++ b/src/main/runtime/structured-agent-session-lifecycle-delivery.ts @@ -3,8 +3,9 @@ // Exit recovery runs on one chain so teardown can drain it: exit callbacks arrive from child // process tasks, and a fire-and-forget one could otherwise append after the host flushed and // removed its journal directory. That chain orders nothing across sessions, and a recovery on it -// can run a whole reacquisition, so `started` stays off it: it takes only its own session's -// serialized step, and is tracked here so the same drain still waits for it. +// can run a whole reacquisition, so `started` and the end of a close the host asked for stay off +// it: each takes only its own session's serialized step, and is tracked here so the same drain +// still waits for it. import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' @@ -34,8 +35,8 @@ export function createStructuredAgentSessionLifecycleDelivery(input: { } return { deliver: (event) => { - if (event.type === 'started') { - // Called now, so the step is queued on its session ahead of any later exit of that child. + if (event.type === 'started' || event.cause === 'requested-close') { + // Called now, so the step is queued on its own session's lane, behind nothing of another's. const settling = settle(event) settlingStarts.add(settling) void settling.finally(() => settlingStarts.delete(settling)) From e97710fd8ae5f9ebd798a7a916c85a62b1894c0c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:39:47 -0700 Subject: [PATCH 06/22] fix(native-chat): an exit's lease release lands whatever the length of its reason A crash's reason can carry kilobytes of the provider's stderr, and a lease whose death detail is over 512 characters fails the store's own check. The exit handler cut it, but the release a start or the chat handle's close re-derives did not, so after a crash whose own release failed every message was refused as not resumable until restart. The record's builder now cuts the detail to the record's bound, so no writer can hand it one too long. --- .../structured-agent-session-child-exit.ts | 3 +-- ...ssion-claude-stop-exit-ends-record.test.ts | 22 +++++++++++++++++++ ...gent-session-surface-release-transition.ts | 10 +++++++-- src/shared/agent-session-record.ts | 4 +++- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 8a470f209187..29fd78812f04 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -19,7 +19,6 @@ import { import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' import { captureUnfinishedStructuredAgentSessionWork, - MAX_UNEXPECTED_EXIT_REASON_CHARS, settleStructuredAgentSessionDeadGeneration, type DeadGenerationJournal, unfinishedStructuredAgentSessionWorkWasInterrupted @@ -182,7 +181,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< expectedFence: child.fence, now: context.now(), exitObservedAt: observedAt, - exitReason: exit.reason.slice(0, MAX_UNEXPECTED_EXIT_REASON_CHARS) + exitReason: exit.reason }) released = true } catch (error) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index 91ac994d4a74..8c187c0f85ad 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -387,6 +387,28 @@ it('sends after a proven exit whose resume-point write and lease release both fa await resumedWith(connection, 'Carry on.') }) +// A crash's reason can carry kilobytes of stderr; the release a start re-derives must still land. +it('sends after a crash with a long reason whose own lease release failed', async () => { + const connection = claude.connections[0]! + const transition = store.transitionHandoff.bind(store) + vi.spyOn(store, 'transitionHandoff') + .mockImplementationOnce(async () => { + throw new Error('store unavailable') + }) + .mockImplementation(transition) + + connection.handlers.onExit?.( + new Error(`claude stream-json exited (code 1): ${'stack frame\n'.repeat(700)}`) + ) + await eventually(() => expect(child()).toBeNull()) + await eventually(() => expect(scopes()).toContain('exit-owner-release')) + expect(lease()?.claimStatus).toBe('live') + + await send('Carry on.') + await resumedWith(connection, 'Carry on.') + expect(scopes()).not.toContain('ended-child-lease-release') +}) + it('rejects a message whose start meets a close still unverifiable, and starts nothing beside it', async () => { const connection = await stopWithUnprovenClose(2) diff --git a/src/main/runtime/agent-session-surface-release-transition.ts b/src/main/runtime/agent-session-surface-release-transition.ts index fe98a8705248..1099e8d6c045 100644 --- a/src/main/runtime/agent-session-surface-release-transition.ts +++ b/src/main/runtime/agent-session-surface-release-transition.ts @@ -9,7 +9,10 @@ // holding the stopped owner's fence is refused as stale rather than acting on its successor. import { agentSessionRefusalError } from '../../shared/agent-session-wire-refusals' -import type { AgentSessionRecord } from '../../shared/agent-session-record' +import { + MAX_AGENT_SESSION_DEATH_DETAIL_CHARS, + type AgentSessionRecord +} from '../../shared/agent-session-record' import { nextAgentSessionFence } from '../../shared/agent-session-next-fence' import { assertFence, withLease } from './agent-session-lease-transitions' import type { AgentSessionRecordStore } from './agent-session-record-store' @@ -50,7 +53,10 @@ export function releaseAgentSessionOwnerAfterSurfaceClose(args: { lastRenewedAt: args.now, deathEvidence: { kind: 'exit-observed', - detail: args.exitReason ?? 'the last surface holding this session released it', + // A provider's exit reason can carry kilobytes of stderr; a longer detail fails the write. + detail: args.exitReason + ? args.exitReason.slice(0, MAX_AGENT_SESSION_DEATH_DETAIL_CHARS) + : 'the last surface holding this session released it', observedAt: args.exitObservedAt ?? args.now, ownerFence: record.lease.runtimeFence } diff --git a/src/shared/agent-session-record.ts b/src/shared/agent-session-record.ts index b8280cf332da..a9d563392f02 100644 --- a/src/shared/agent-session-record.ts +++ b/src/shared/agent-session-record.ts @@ -157,6 +157,8 @@ export type AgentSessionOptionsReplacement = { } const MAX_ID_LENGTH = 512 +/** A death evidence's `detail` past this fails a load, so whoever writes one cuts it here. */ +export const MAX_AGENT_SESSION_DEATH_DETAIL_CHARS = MAX_ID_LENGTH const MAX_PATH_LENGTH = 4096 const MAX_LAUNCH_ENV_ENTRIES = 256 const MAX_LAUNCH_ENV_VALUE_LENGTH = 65_536 @@ -290,7 +292,7 @@ function isAgentSessionDeathEvidence(value: unknown): value is AgentSessionDeath (evidence.kind === 'exit-observed' || evidence.kind === 'pid-absent' || evidence.kind === 'identity-mismatch') && - isBoundedString(evidence.detail, MAX_ID_LENGTH) && + isBoundedString(evidence.detail, MAX_AGENT_SESSION_DEATH_DETAIL_CHARS) && typeof observedAt === 'number' && Number.isSafeInteger(observedAt) && observedAt >= 0 && From ce52df90617828f284385b63aaff53d2afff2d58 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:40:06 -0700 Subject: [PATCH 07/22] fix(claude): a proven close waits at most 2 s for the output it already wrote Once the root's exit is proven, the close still waited for the SDK's output reader to end. Something outside the process tree that holds the output open would keep that close, and every send, Stop and quit joining it, waiting with no bound. The wait is now bounded; past it the close resolves as proven and the open output is logged. --- ...laude-stream-json-connection-close.test.ts | 49 +++++++++++++++++++ .../claude/claude-stream-json-connection.ts | 17 ++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/main/claude/claude-stream-json-connection-close.test.ts b/src/main/claude/claude-stream-json-connection-close.test.ts index f5bbc35d2698..aa4beb4eed96 100644 --- a/src/main/claude/claude-stream-json-connection-close.test.ts +++ b/src/main/claude/claude-stream-json-connection-close.test.ts @@ -4,6 +4,7 @@ import type { ChildProcessWithoutNullStreams } from 'node:child_process' import { describe, expect, it, vi } from 'vitest' import type { query } from '@anthropic-ai/claude-agent-sdk' import { + CLAUDE_READER_DRAIN_AFTER_EXIT_MS, openClaudeStreamJsonConnection, type ClaudeStreamJsonLaunch } from './claude-stream-json-connection' @@ -162,6 +163,54 @@ describe('Claude stream-json close ordering', () => { expect(next).toHaveBeenCalledOnce() }) + // Whatever holds the output open past a proven exit (a process that escaped the tree) must not + // keep every send and Stop that joins this close waiting. + it('resolves a proven close though the output never ends', async () => { + mocks.refresh.mockReset() + mocks.proveClaudeChildExit.mockReset() + mocks.refresh.mockResolvedValue(undefined) + mocks.proveClaudeChildExit.mockResolvedValue(true) + const child = fakeChild() + const next = vi.fn(() => new Promise>>(() => {})) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This injected query exercises only the async iterator used by the connection. + const queryImpl = ((params: Parameters[0]) => { + params.options?.spawnClaudeCodeProcess?.({ + command: 'claude', + args: [], + env: {}, + signal: new AbortController().signal + }) + return { + [Symbol.asyncIterator]: () => ({ next }) + } + }) as unknown as typeof query + const connection = await openClaudeStreamJsonConnection( + { pathToClaudeCodeExecutable: 'claude', options: {}, cwd: '/work/repo' }, + {}, + () => child, + queryImpl + ) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.useFakeTimers() + try { + let closed: boolean | undefined + void connection.close().then((value) => { + closed = value + }) + await vi.advanceTimersByTimeAsync(CLAUDE_READER_DRAIN_AFTER_EXIT_MS - 1) + expect(closed).toBeUndefined() + await vi.advanceTimersByTimeAsync(1) + expect(closed).toBe(true) + expect(warn).toHaveBeenCalledWith( + '[claude-stream-json] output still open after the proven exit:', + expect.anything() + ) + } finally { + vi.useRealTimers() + warn.mockRestore() + } + }) + it('reports a root that exits after its close came back unproven as that close ending', async () => { mocks.refresh.mockReset() mocks.proveClaudeChildExit.mockReset() diff --git a/src/main/claude/claude-stream-json-connection.ts b/src/main/claude/claude-stream-json-connection.ts index b7c50646fb46..d6511a998630 100644 --- a/src/main/claude/claude-stream-json-connection.ts +++ b/src/main/claude/claude-stream-json-connection.ts @@ -15,6 +15,7 @@ import { type ClaudeControlSurface } from './claude-agent-sdk-control-requests' import { createClaudeChildTreeReaper, proveClaudeChildExit } from './claude-agent-sdk-exit-proof' +import { withTimeout } from '../../shared/promise-timeout-fallback' import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification' import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn' import { @@ -25,6 +26,10 @@ import type { ClaudeStructuredSdkOptions } from './claude-structured-launch-reso export { ClaudeControlRequestError } +/** How long a proven close waits for messages already written before the exit; whatever still + * holds the output open past it is no reason to keep the close unresolved. */ +export const CLAUDE_READER_DRAIN_AFTER_EXIT_MS = 2_000 + /** * The SDK is loaded at the structured-Claude boundary rather than by this module's * import. The ordinary runtime's class graph statically reaches this file, and the @@ -33,6 +38,7 @@ export { ClaudeControlRequestError } * path never opted into, and a missing SDK would fail runtime startup. Memoized, * so a session pays the import once per process rather than once per connection. */ + let claudeAgentSdk: Promise | null = null function loadClaudeAgentSdk(): Promise { @@ -332,7 +338,16 @@ export async function openClaudeStreamJsonConnection( closePromise = null return false } - await readerDone + const drained = await withTimeout( + readerDone.then(() => true), + CLAUDE_READER_DRAIN_AFTER_EXIT_MS, + false + ) + if (!drained) { + console.warn('[claude-stream-json] output still open after the proven exit:', { + pid: spawner.pid + }) + } return true })() return closePromise From 1d315b8e09b841050cf04c218bdfa28e52502968 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:40:21 -0700 Subject: [PATCH 08/22] fix(codex): an exit reported inside Orca's close keeps the reason Orca closed it for The connection reports the app-server's exit inside the close that ends it, so that report ended every Codex close and replaced the close's own reason (for example, a provider frame that could not be recorded) with the connection's stderr text in the ended record and the lease's exit evidence. The session now records Orca's close with its reason, and the exit it ends keeps that reason. The test connection reports its exit inside close the way the real one does. --- src/main/codex/codex-close-turn-end.test.ts | 1 - .../codex/codex-requested-close-turn-timing.test.ts | 1 - .../codex-structured-session-adapter-fixture.ts | 12 +++++++++--- .../codex/codex-structured-session-close.test.ts | 2 +- src/main/codex/codex-structured-session-close.ts | 12 ++++++++---- .../codex-structured-session-options-catalog.test.ts | 1 - .../codex/codex-structured-session-options.test.ts | 1 - src/main/codex/codex-structured-session-state.ts | 10 ++++------ 8 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/main/codex/codex-close-turn-end.test.ts b/src/main/codex/codex-close-turn-end.test.ts index 1e3475301bf0..43865f423758 100644 --- a/src/main/codex/codex-close-turn-end.test.ts +++ b/src/main/codex/codex-close-turn-end.test.ts @@ -58,7 +58,6 @@ function sessionWithRunningTurn() { }, backgroundTasks: new CodexBackgroundTaskTracker('thread-1'), ended: false, - requestedClose: false, fence: 7, acquisitionGeneration: 'generation-1', threadId: 'thread-1', diff --git a/src/main/codex/codex-requested-close-turn-timing.test.ts b/src/main/codex/codex-requested-close-turn-timing.test.ts index 45754f9f8042..5d110cb6e72c 100644 --- a/src/main/codex/codex-requested-close-turn-timing.test.ts +++ b/src/main/codex/codex-requested-close-turn-timing.test.ts @@ -53,7 +53,6 @@ describe('requested-close durable turn timing', () => { }, backgroundTasks: new CodexBackgroundTaskTracker('thread-1'), ended: false, - requestedClose: false, fence: 7, acquisitionGeneration: 'generation-1', threadId: 'thread-1', diff --git a/src/main/codex/codex-structured-session-adapter-fixture.ts b/src/main/codex/codex-structured-session-adapter-fixture.ts index f2b4e102c355..37d434ef1a41 100644 --- a/src/main/codex/codex-structured-session-adapter-fixture.ts +++ b/src/main/codex/codex-structured-session-adapter-fixture.ts @@ -50,7 +50,7 @@ export function fakeCodex(routes: Record = {}): { routes: Record } { const connections: FakeConnection[] = [] - const openConnection = (async (launch, handlers = {}) => { + const openConnection: typeof openCodexAppServerConnection = async (launch, handlers = {}) => { const connection: FakeConnection = { launch, handlers, @@ -67,15 +67,21 @@ export function fakeCodex(routes: Record = {}): { notify: () => {}, respond: (id, result) => connection.replies.push({ id, result }), respondWithError: (id, code, message) => connection.replies.push({ id, code, message }), + // As the real connection: the root's exit is reported, once, inside the close that ends it. close: async () => { connection.closeCount += 1 - connection.closed = true + if (!connection.closed) { + connection.closed = true + handlers.onExit?.(new Error('codex app-server connection ended: killed'), { + expected: true + }) + } return true } } connections.push(connection) return connection - }) as typeof openCodexAppServerConnection + } routes['thread/start'] ??= () => ({ thread: { id: THREAD_ID, path: '/rollouts/abc.jsonl' }, model: 'gpt-live', diff --git a/src/main/codex/codex-structured-session-close.test.ts b/src/main/codex/codex-structured-session-close.test.ts index 73004e43c344..79e4ea977a36 100644 --- a/src/main/codex/codex-structured-session-close.test.ts +++ b/src/main/codex/codex-structured-session-close.test.ts @@ -96,7 +96,7 @@ describe('Codex structured session close lifecycle', () => { connection, backgroundTasks: new CodexBackgroundTaskTracker('thread-1'), ended: false, - requestedClose, + ...(requestedClose ? { orcaClose: { requested: true, reason: new Error('closed') } } : {}), fence: 7, acquisitionGeneration: 'generation-1', threadId: THREAD, diff --git a/src/main/codex/codex-structured-session-close.ts b/src/main/codex/codex-structured-session-close.ts index 3ab803101aa0..465d7801fec9 100644 --- a/src/main/codex/codex-structured-session-close.ts +++ b/src/main/codex/codex-structured-session-close.ts @@ -32,12 +32,13 @@ export function handleCodexSessionExit(input: { const event: StructuredAgentSessionEndedEvent = { type: 'ended', sessionId: input.sessionId, - reason: input.error.message, + // The connection reports the exit inside the close it ends; the close's own reason is the why. + reason: ((input.closedByOrca && session.orcaClose?.reason) || input.error).message, // Only the child's own exit blames Codex; a close Orca made, for any reason, is Orca's. failure: input.closedByOrca ? agentSessionFailureFact('hostFault') : agentSessionFailureFact('providerExited', { detail: providerDiagnosticOf(input.error) }), - cause: session.requestedClose ? 'requested-close' : 'unexpected-exit', + cause: session.orcaClose?.requested ? 'requested-close' : 'unexpected-exit', fence: session.fence, acquisitionGeneration: session.acquisitionGeneration, observedAt: session.exitObservedAt @@ -92,7 +93,10 @@ export async function closeCodexPublishedSession( } // Sink-failure recovery force-closes the child but must preserve the // observed-exit cause so host lease settlement runs as an unexpected death. - session.requestedClose = options?.requestedClose ?? true + session.orcaClose = { + requested: options?.requestedClose ?? true, + reason: options?.unexpectedReason ?? new Error('codex session closed') + } // Keep the session indexed until the child exit is observed. A timeout or // failed kill must leave the live connection available for a safe retry. const exited = await session.connection.close() @@ -104,7 +108,7 @@ export async function closeCodexPublishedSession( sessions, sessionId, connection: session.connection, - error: options?.unexpectedReason ?? new Error('codex session closed'), + error: session.orcaClose.reason, closedByOrca: true, prompts: session.prompts, ...(onEvent ? { onEvent } : {}), diff --git a/src/main/codex/codex-structured-session-options-catalog.test.ts b/src/main/codex/codex-structured-session-options-catalog.test.ts index aac9e2946cf7..57c121c156dc 100644 --- a/src/main/codex/codex-structured-session-options-catalog.test.ts +++ b/src/main/codex/codex-structured-session-options-catalog.test.ts @@ -53,7 +53,6 @@ function storeSession( }, backgroundTasks: new CodexBackgroundTaskTracker('thread-1'), ended: false, - requestedClose: false, fence: 1, acquisitionGeneration: 'generation-1', threadId: 'thread-1', diff --git a/src/main/codex/codex-structured-session-options.test.ts b/src/main/codex/codex-structured-session-options.test.ts index d99ecc0305e1..e07f0b4c8847 100644 --- a/src/main/codex/codex-structured-session-options.test.ts +++ b/src/main/codex/codex-structured-session-options.test.ts @@ -27,7 +27,6 @@ function optionSession(request: CodexAppServerConnection['request']): CodexSessi }, backgroundTasks: new CodexBackgroundTaskTracker('thread-1'), ended: false, - requestedClose: false, fence: 1, acquisitionGeneration: 'generation-1', threadId: 'thread-1', diff --git a/src/main/codex/codex-structured-session-state.ts b/src/main/codex/codex-structured-session-state.ts index 74246b6a7f17..78e7291fd742 100644 --- a/src/main/codex/codex-structured-session-state.ts +++ b/src/main/codex/codex-structured-session-state.ts @@ -108,7 +108,9 @@ export type CodexSession = { ended: boolean /** First observed child exit survives rejected settlement admission. */ exitObservedAt?: number - requestedClose: boolean + /** The close Orca began for this child: asked for, or forced as a death, and why. Whatever ends + * the child after it (that close, or the exit the connection reports meanwhile) keeps this. */ + orcaClose?: { requested: boolean; reason: Error } fence: number acquisitionGeneration: string threadId: string @@ -149,13 +151,9 @@ export function mintCodexAcquisitionGeneration(deps: CodexStructuredSessionAdapt export function codexSessionLifecycle( fence: number, acquisitionGeneration: string -): Pick< - CodexSession, - 'ended' | 'requestedClose' | 'fence' | 'acquisitionGeneration' | 'turnOpenWaits' -> { +): Pick { return { ended: false, - requestedClose: false, fence, acquisitionGeneration, turnOpenWaits: createCodexTurnOpenWaits() From 31819dbcb19645116635c8d33b42af79f29d97a2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:40:36 -0700 Subject: [PATCH 09/22] fix(native-chat): quit stops delivery before it drains exit recovery Every exit now wakes delivery, and teardown drained exit recovery before it stopped delivery, so an exit settled in that window could start a fresh agent that teardown then killed. Teardown stops delivery first; queued messages wait for the next launch. --- .../journal-host-database.test.ts | 3 ++- .../structured-agent-session-host.ts | 5 +++- ...red-agent-session-runtime-teardown.test.ts | 27 +++++++++++++++++++ ...ructured-agent-session-runtime-teardown.ts | 3 +++ 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 src/main/runtime/structured-agent-session-runtime-teardown.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-host-database.test.ts b/src/main/native-chat/agent-session-journal/journal-host-database.test.ts index 84ff9775022b..2b4e4a0819a7 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database.test.ts @@ -232,8 +232,9 @@ describe('quit', () => { const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) const database = openTestJournalHostDatabase(root) const installed = { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: teardown calls only `flushAllStreamedEvents` on the host. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: teardown calls only `stopDelivery` and `flushAllStreamedEvents` on the host. host: { + stopDelivery: () => {}, flushAllStreamedEvents: async () => { // A child's last row, delivered while quit is draining its sink. await new Promise((resolve) => setTimeout(resolve, 10)) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index bdb82c8fadb1..3502a2dcae9c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -258,8 +258,11 @@ export class StructuredAgentSessionHost { // Trigger inlined rather than imported: `AgentSessionResumeTrigger` in shared is the canonical // type, and this file has no line budget left for the import. + /** Quit: no exit or recovery settled after this starts a child or hands a message over. */ + stopDelivery = (): void => this.conversationDelivery.dispose() + async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise { - this.conversationDelivery.dispose() + this.stopDelivery() await flushStructuredAgentSessionHost({ ...this.lifetimeContext(), idleSweep: this.lifetime, diff --git a/src/main/runtime/structured-agent-session-runtime-teardown.test.ts b/src/main/runtime/structured-agent-session-runtime-teardown.test.ts new file mode 100644 index 000000000000..767d0716c445 --- /dev/null +++ b/src/main/runtime/structured-agent-session-runtime-teardown.test.ts @@ -0,0 +1,27 @@ +import { expect, it } from 'vitest' +import { tearDownRuntime, type InstalledRuntime } from './structured-agent-session-runtime-teardown' + +// An exit that settles while teardown drains recovery wakes delivery; a fresh child started then +// would only be killed by the teardown after it. +it('stops delivery before it drains the recovery of exits already observed', async () => { + const order: string[] = [] + const installed: InstalledRuntime = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: teardown calls only `stopDelivery` and `flushAllStreamedEvents` on the host. + host: { + stopDelivery: () => order.push('stop-delivery'), + flushAllStreamedEvents: async () => { + order.push('flush') + } + } as never, + adapter: { closeAll: async () => undefined }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: teardown calls only `close` on the database. + journalDatabase: { close: () => order.push('close-database') } as never, + waitForRecovery: async () => { + order.push('wait-for-recovery') + } + } + + await tearDownRuntime(installed, 'quit') + + expect(order.slice(0, 2)).toEqual(['stop-delivery', 'wait-for-recovery']) +}) diff --git a/src/main/runtime/structured-agent-session-runtime-teardown.ts b/src/main/runtime/structured-agent-session-runtime-teardown.ts index 47d93e028307..3275939a4fa0 100644 --- a/src/main/runtime/structured-agent-session-runtime-teardown.ts +++ b/src/main/runtime/structured-agent-session-runtime-teardown.ts @@ -35,6 +35,9 @@ export async function tearDownRuntime( installed: InstalledRuntime, trigger: AgentSessionResumeTrigger ): Promise { + // An exit settled while recovery drains wakes delivery, which would start a fresh child for + // teardown to kill; queued messages wait for the next launch instead. + installed.host.stopDelivery() // Drain an in-flight recovery before stopping children; recovery may still // be writing lifecycle rows or acquiring a replacement child. await installed.waitForRecovery() From aa7928c29a069be814afebe513cd748b81edf933 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:40:46 -0700 Subject: [PATCH 10/22] docs(native-chat): the unverifiable-exit refusal no longer names a caller's wait The caller's bounded wait was removed; the comment describes the close as it is now. --- src/shared/agent-session-refusal-details.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/agent-session-refusal-details.ts b/src/shared/agent-session-refusal-details.ts index 225a0861d46e..b56ca5fbeaaa 100644 --- a/src/shared/agent-session-refusal-details.ts +++ b/src/shared/agent-session-refusal-details.ts @@ -67,8 +67,8 @@ export const AGENT_SESSION_REFUSAL_REASONS = { 'notResumable', 'noProviderChild', 'conversationHeldElsewhere', - /** The close a stop began could not prove its child gone within the caller's wait: that child - * takes no input and none starts beside it. Sent with `ownerVerdict: 'unverifiable'`. */ + /** The close a stop began could not prove its child gone: that child takes no input and none + * starts beside it. Sent with `ownerVerdict: 'unverifiable'`. */ 'previousExitUnverifiable' ], agent_session_conflict: [ From 121150b0cd8362bed005dc9f0de42ca2c38c5c85 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:31:37 -0700 Subject: [PATCH 11/22] fix(native-chat): a stop whose kill did not take is logged, and the next ask kills again When a close's kill leaves the agent's root running, the host now logs it. Tests pin what a later ask does: each connection runs its whole stop again (Codex sends SIGKILL a second time), refuses input meanwhile, and proves the exit once the kill takes. --- ...laude-stream-json-connection-close.test.ts | 36 +++++++++++++++++++ .../codex/codex-app-server-connection.test.ts | 27 ++++++++++++++ .../structured-agent-session-child-close.ts | 28 ++++++++++----- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/src/main/claude/claude-stream-json-connection-close.test.ts b/src/main/claude/claude-stream-json-connection-close.test.ts index aa4beb4eed96..348601bbd489 100644 --- a/src/main/claude/claude-stream-json-connection-close.test.ts +++ b/src/main/claude/claude-stream-json-connection-close.test.ts @@ -163,6 +163,42 @@ describe('Claude stream-json close ordering', () => { expect(next).toHaveBeenCalledOnce() }) + // The ladder (stdin end, SIGTERM, SIGKILL) runs again for the next ask; nothing reuses the root. + it('runs the stop ladder again on a close after an unproven attempt, refusing input meanwhile', async () => { + mocks.refresh.mockReset() + mocks.proveClaudeChildExit.mockReset() + mocks.refresh.mockResolvedValue(undefined) + mocks.proveClaudeChildExit.mockResolvedValueOnce(false).mockResolvedValueOnce(true) + const child = fakeChild() + const next = vi.fn(async (): Promise>> => ({ + value: undefined, + done: true + })) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This injected query exercises only the async iterator used by the connection. + const queryImpl = ((params: Parameters[0]) => { + params.options?.spawnClaudeCodeProcess?.({ + command: 'claude', + args: [], + env: {}, + signal: new AbortController().signal + }) + return { + [Symbol.asyncIterator]: () => ({ next }) + } + }) as unknown as typeof query + const connection = await openClaudeStreamJsonConnection( + { pathToClaudeCodeExecutable: 'claude', options: {}, cwd: '/work/repo' }, + {}, + () => child, + queryImpl + ) + + await expect(connection.close()).resolves.toBe(false) + await expect(connection.send({ type: 'user' })).rejects.toThrow() + await expect(connection.close()).resolves.toBe(true) + expect(mocks.proveClaudeChildExit).toHaveBeenCalledTimes(2) + }) + // Whatever holds the output open past a proven exit (a process that escaped the tree) must not // keep every send and Stop that joins this close waiting. it('resolves a proven close though the output never ends', async () => { diff --git a/src/main/codex/codex-app-server-connection.test.ts b/src/main/codex/codex-app-server-connection.test.ts index 962270423e33..fa87062d724e 100644 --- a/src/main/codex/codex-app-server-connection.test.ts +++ b/src/main/codex/codex-app-server-connection.test.ts @@ -526,6 +526,33 @@ describe('openCodexAppServerConnection', () => { await expect(connection.close()).resolves.toBe(true) }) + // A root that outlived one kill is killed again by the next ask, which then proves it gone. + it('kills the root again on a close after an unproven attempt', async () => { + vi.useFakeTimers() + const { child, spawnImpl } = stubChild({ exitOnStdinEnd: false }) + answerInitialize(child) + const connection = await openCodexAppServerConnection( + { command: 'codex', args: ['app-server'] }, + {}, + spawnImpl + ) + + const first = connection.close() + await vi.advanceTimersByTimeAsync(GRACEFUL_EXIT_MS + 3_500) + await expect(first).resolves.toBe(false) + child.kill.mockImplementation((signal) => { + if (signal === 'SIGKILL') { + setTimeout(() => child.emit('exit', null, 'SIGKILL'), 10) + } + return true + }) + + const second = connection.close() + await vi.advanceTimersByTimeAsync(GRACEFUL_EXIT_MS + 3_500) + await expect(second).resolves.toBe(true) + expect(child.kill.mock.calls.filter(([signal]) => signal === 'SIGKILL')).toHaveLength(2) + }) + it.each([1_090_188, 2_900_090])( 'accepts a realistic %i-byte escaped command completion and keeps processing', async (frameBytes) => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index d9232d142124..a8c08201394b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -77,14 +77,26 @@ function closeProviderRoot( sessionId, error }) - ).catch((error: unknown) => { - logger.warn("closing the agent's process did not prove it exited", { - scope: 'provider-close', - sessionId, - error - }) - return false - }) + ).then( + (exited) => { + if (!exited) { + // The adapter asked, then killed; the root is still there. The next ask kills again. + logger.warn("the agent's process did not exit after Orca stopped and killed it", { + scope: 'provider-close-unproven', + sessionId + }) + } + return exited + }, + (error: unknown) => { + logger.warn("closing the agent's process did not prove it exited", { + scope: 'provider-close', + sessionId, + error + }) + return false + } + ) } /** Whether the lease still names the child this host last proved gone, unreleased: only that From 1f4a19c1311e89748779225546e0f0bfdb4f81ef Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:31:44 -0700 Subject: [PATCH 12/22] fix(native-chat): a start refused over the old process says Orca couldn't stop it The host reaches an unverifiable verdict only after its own kill left the agent's root running, on the machine that runs the agent, so the sentence now says that: "Orca couldn't stop {agent}'s previous process." The refusal reason, failure kind and wire shapes are unchanged. The host test also checks the failed kill is logged. --- ...ructured-agent-session-claude-stop-exit-ends-record.test.ts | 3 ++- .../structured-agent-session-codex-stop-row.test.ts | 3 +-- src/renderer/src/i18n/en-runtime-required.json | 2 +- src/renderer/src/i18n/locales/en.json | 2 +- src/renderer/src/i18n/locales/es.json | 2 +- src/renderer/src/i18n/locales/fr.json | 2 +- src/renderer/src/i18n/locales/ja.json | 2 +- src/renderer/src/i18n/locales/ko.json | 2 +- src/renderer/src/i18n/locales/zh.json | 2 +- src/shared/agent-session-failure-copy.ts | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index 8c187c0f85ad..0aeac6c7832c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -417,7 +417,7 @@ it('rejects a message whose start meets a close still unverifiable, and starts n await eventually(async () => expect((await submission(next))?.dispatchState).toBe('rejected')) const rejected = (await submission(next))! expect(rejected.reason).toBe( - "Orca couldn't confirm Claude's previous process ended. Send your message to try again." + "Orca couldn't stop Claude's previous process. Send your message to try again." ) expect(rejected.rejection).toMatchObject({ kind: 'restartFailed', @@ -432,6 +432,7 @@ it('rejects a message whose start meets a close still unverifiable, and starts n expect(claude.connections).toHaveLength(1) expect(wrote(connection, 'Carry on.')).toBe(false) expect(child()?.close).toBeDefined() + expect(scopes()).toContain('provider-close-unproven') }) it('refuses an option change while the close stays unverifiable, never writing to the old child', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts index c92b23a49b9b..a59f7eac346f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts @@ -497,8 +497,7 @@ describe('a message after a Codex Stop whose exit was unproven', () => { ) ).toMatchObject({ dispatchState: 'rejected', - reason: - "Orca couldn't confirm Codex's previous process ended. Send your message to try again." + reason: "Orca couldn't stop Codex's previous process. Send your message to try again." }) ) expect(codex.connections).toHaveLength(1) diff --git a/src/renderer/src/i18n/en-runtime-required.json b/src/renderer/src/i18n/en-runtime-required.json index 2ce3ec2b2420..b29a32458333 100644 --- a/src/renderer/src/i18n/en-runtime-required.json +++ b/src/renderer/src/i18n/en-runtime-required.json @@ -2759,7 +2759,7 @@ "notDelivered": "This message was not delivered.", "notDeliveredSendAgain": "This message was not delivered. Send it again to continue.", "notSignedIn": "{{agent}} is not signed in for the selected account.", - "previousExitUnverifiable": "Orca couldn't confirm {{agent}}'s previous process ended.", + "previousExitUnverifiable": "Orca couldn't stop {{agent}}'s previous process.", "providerExitedRejection": "{{agent}} stopped before this message was sent.", "providerExitedRow": "{{agent}} stopped while this response was in progress. You can continue in this conversation.", "providerRateLimited": "{{agent}} is rate-limited and retrying.", diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 9d0a16114491..ed1cb65313ad 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -17561,7 +17561,7 @@ "providerRateLimited": "{{agent}} is rate-limited and retrying.", "providerRetrying": "{{agent}} hit a temporary problem and is retrying.", "providerRetryingQuoted": "{{agent}} is retrying: {{detail}}.", - "previousExitUnverifiable": "Orca couldn't confirm {{agent}}'s previous process ended." + "previousExitUnverifiable": "Orca couldn't stop {{agent}}'s previous process." }, "writeNotice": { "notDoneReadHistory": "This chat's history couldn't be loaded.", diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 95d780a4ac47..977af682a035 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -14999,7 +14999,7 @@ "providerRateLimited": "{{agent}} alcanzó un límite de solicitudes y está reintentando.", "providerRetrying": "{{agent}} tuvo un problema temporal y está reintentando.", "providerRetryingQuoted": "{{agent}} está reintentando: {{detail}}.", - "previousExitUnverifiable": "Orca no pudo confirmar que el proceso anterior de {{agent}} terminó." + "previousExitUnverifiable": "Orca no pudo detener el proceso anterior de {{agent}}." }, "writeNotice": { "notDoneReadHistory": "No se pudo cargar el historial de este chat.", diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 2d1a996c67db..8ee77c754f5c 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -17465,7 +17465,7 @@ "providerRateLimited": "{{agent}} est limité en débit et réessaie.", "providerRetrying": "{{agent}} a rencontré un problème temporaire et réessaie.", "providerRetryingQuoted": "{{agent}} réessaie : {{detail}}.", - "previousExitUnverifiable": "Orca n'a pas pu confirmer que le processus précédent de {{agent}} s'est arrêté." + "previousExitUnverifiable": "Orca n'a pas pu arrêter le processus précédent de {{agent}}." }, "writeNotice": { "notDoneReadHistory": "L'historique de ce chat n'a pas pu être chargé.", diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index a027e5e34e7d..12782e390ed6 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -17401,7 +17401,7 @@ "providerRateLimited": "{{agent}} はレート制限を受けているため、再試行しています。", "providerRetrying": "{{agent}} で一時的な問題が発生したため、再試行しています。", "providerRetryingQuoted": "{{agent}} は再試行しています: {{detail}}。", - "previousExitUnverifiable": "Orca は {{agent}} の前のプロセスが終了したことを確認できませんでした。" + "previousExitUnverifiable": "Orca は {{agent}} の前のプロセスを停止できませんでした。" }, "writeNotice": { "notDoneReadHistory": "このチャットの履歴を読み込めませんでした。", diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 7c01fde23971..7b25344fe952 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -17401,7 +17401,7 @@ "providerRateLimited": "{{agent}}이(가) 속도 제한에 걸려 다시 시도하고 있습니다.", "providerRetrying": "{{agent}}에 일시적인 문제가 발생하여 다시 시도하고 있습니다.", "providerRetryingQuoted": "{{agent}}이(가) 다시 시도하고 있습니다: {{detail}}.", - "previousExitUnverifiable": "Orca가 {{agent}}의 이전 프로세스가 종료되었는지 확인하지 못했습니다." + "previousExitUnverifiable": "Orca가 {{agent}}의 이전 프로세스를 중지하지 못했습니다." }, "writeNotice": { "notDoneReadHistory": "이 채팅의 기록을 불러오지 못했습니다.", diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index bfb7bd46911c..721d1acae7b3 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -17366,7 +17366,7 @@ "providerRateLimited": "{{agent}} 已被限流，正在重试。", "providerRetrying": "{{agent}} 遇到临时问题，正在重试。", "providerRetryingQuoted": "{{agent}} 正在重试：{{detail}}。", - "previousExitUnverifiable": "Orca 无法确认 {{agent}} 的上一个进程已结束。" + "previousExitUnverifiable": "Orca 无法停止 {{agent}} 的上一个进程。" }, "writeNotice": { "notDoneReadHistory": "无法加载此聊天的历史记录。", diff --git a/src/shared/agent-session-failure-copy.ts b/src/shared/agent-session-failure-copy.ts index 04fa33c4b2bb..f40ffcfc1d2c 100644 --- a/src/shared/agent-session-failure-copy.ts +++ b/src/shared/agent-session-failure-copy.ts @@ -81,7 +81,7 @@ export const AGENT_SESSION_FAILURE_COPY = { providerRateLimited: '{{agent}} is rate-limited and retrying.', providerRetrying: '{{agent}} hit a temporary problem and is retrying.', providerRetryingQuoted: '{{agent}} is retrying: {{detail}}.', - previousExitUnverifiable: "Orca couldn't confirm {{agent}}'s previous process ended." + previousExitUnverifiable: "Orca couldn't stop {{agent}}'s previous process." } as const export type AgentSessionFailureCopyId = keyof typeof AGENT_SESSION_FAILURE_COPY From 54512de37bfbff44c3263d84e6c4d5999703b806 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:43:24 -0700 Subject: [PATCH 13/22] docs(native-chat): an unverifiable close verdict is a root that survived the kill The host's close runs where the agent runs, so lost contact never yields this verdict; the comment no longer says it does. --- .../structured-agent-session-child-close.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index a8c08201394b..402c5677e9ef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -15,8 +15,8 @@ import { stopAgentSessionProviderRoot } from './structured-agent-session-provide import { releaseStoredStructuredAgentSessionOwnerAfterExit } from './structured-agent-session-lease-release' import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' -/** What a caller learned about the child's exit: proven, or not. A close that ran out of its own - * escalation, or a host Orca lost contact with, reads `unverifiable`, never `exited`. */ +/** What a caller learned about the child's exit: proven, or not. A root still there after the + * close's kill reads `unverifiable`, never `exited`. */ export type StructuredAgentSessionChildCloseVerdict = 'exited' | 'unverifiable' /** Joins the child's close and, once its root's exit is proven, ends the record. */ From 0a98f693d4342893c961ca0150631be88c3f244b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:48:41 -0700 Subject: [PATCH 14/22] fix(native-chat): a kill that did not take is reported once, by whoever met it The log added at the close fired beside a Stop's own failure report for the same event. A stop still reports it through its failure; a send or option change refused over it now logs it at the refusal, the only place it is otherwise invisible. --- .../structured-agent-session-child-close.ts | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index 402c5677e9ef..0022df7c1565 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -53,9 +53,15 @@ export async function joinClosingStructuredAgentSessionChild( if (!child?.close) { return { ok: true } } - return (await joinStructuredAgentSessionChildClose(context, sessionId, child)) === 'exited' - ? { ok: true } - : { ok: false, refusal: previousExitUnverifiableRefusal() } + if ((await joinStructuredAgentSessionChildClose(context, sessionId, child)) === 'exited') { + return { ok: true } + } + // A stop reports this through its own failure; here the refusal is the only trace. + context.deps.logger.warn("the agent's process did not exit after Orca stopped and killed it", { + scope: 'provider-close-unproven', + sessionId + }) + return { ok: false, refusal: previousExitUnverifiableRefusal() } } function closeProviderRoot( @@ -77,26 +83,14 @@ function closeProviderRoot( sessionId, error }) - ).then( - (exited) => { - if (!exited) { - // The adapter asked, then killed; the root is still there. The next ask kills again. - logger.warn("the agent's process did not exit after Orca stopped and killed it", { - scope: 'provider-close-unproven', - sessionId - }) - } - return exited - }, - (error: unknown) => { - logger.warn("closing the agent's process did not prove it exited", { - scope: 'provider-close', - sessionId, - error - }) - return false - } - ) + ).catch((error: unknown) => { + logger.warn("closing the agent's process did not prove it exited", { + scope: 'provider-close', + sessionId, + error + }) + return false + }) } /** Whether the lease still names the child this host last proved gone, unreleased: only that From 1bf8f29e6543a59e15540963643bf2ed787edac5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:55:28 -0700 Subject: [PATCH 15/22] test(native-chat): a second Stop joins a close the first could not prove and retries its kill --- ...nt-session-claude-stop-exit-ends-record.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index c0e11513490f..b0de80e26926 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -290,6 +290,20 @@ it('joins the close a Stop could not prove before the next message, then sends i }) }) +it('has a second Stop join the close the first could not prove, retrying its kill', async () => { + const connection = await stopWithUnprovenClose(1) + + // Nothing was queued, so it withdrew nothing and records no event of its own. + await expect(stop()).resolves.toMatchObject({ ok: true, value: { cancelled: false } }) + expect(connection.closeCount).toBe(2) + expect(child()).toBeNull() + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ + cause: 'user-stop', + rootGone: true + }) + expect(connection.calls.filter((call) => call.subtype === 'interrupt')).toHaveLength(1) +}) + it('has a send made while the close still runs wait for that close, not start another', async () => { const connection = claude.connections[0]! await openTurn(connection) From 8fb8aaf33ea72dd292e64aca79516404c44f8309 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:56:49 -0700 Subject: [PATCH 16/22] fix(native-chat): say a start refused beside an unstopped process plainly The rejection now reads "Couldn't stop {{agent}} from before. Send your message again to try once more." This kind has its own send-again step; every other failure keeps "Send your message to try again." --- ...ssion-claude-stop-exit-ends-record.test.ts | 2 +- ...tured-agent-session-codex-stop-row.test.ts | 2 +- .../agent-session-failure-words-text.ts | 5 +++++ .../src/i18n/en-runtime-required.json | 3 ++- src/renderer/src/i18n/locales/en.json | 3 ++- src/renderer/src/i18n/locales/es.json | 3 ++- src/renderer/src/i18n/locales/fr.json | 3 ++- src/renderer/src/i18n/locales/ja.json | 3 ++- src/renderer/src/i18n/locales/ko.json | 3 ++- src/renderer/src/i18n/locales/zh.json | 3 ++- src/shared/agent-session-failure-copy.ts | 3 ++- .../agent-session-failure-words.test.ts | 22 +++++++++++++++++++ src/shared/agent-session-failure-words.ts | 7 +++--- 13 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index b0de80e26926..cfc2c27ca6be 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -431,7 +431,7 @@ it('rejects a message whose start meets a close still unverifiable, and starts n await eventually(async () => expect((await submission(next))?.dispatchState).toBe('rejected')) const rejected = (await submission(next))! expect(rejected.reason).toBe( - "Orca couldn't stop Claude's previous process. Send your message to try again." + "Couldn't stop Claude from before. Send your message again to try once more." ) expect(rejected.rejection).toMatchObject({ kind: 'restartFailed', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts index a233424c4f73..1dbbac5282d8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts @@ -481,7 +481,7 @@ describe('a message after a Codex Stop whose exit was unproven', () => { ) ).toMatchObject({ dispatchState: 'rejected', - reason: "Orca couldn't stop Codex's previous process. Send your message to try again." + reason: "Couldn't stop Codex from before. Send your message again to try once more." }) ) expect(codex.connections).toHaveLength(1) diff --git a/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts b/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts index 34dab4ab94f6..59c4947c8b4f 100644 --- a/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts +++ b/src/renderer/src/components/native-chat/agent-session-failure-words-text.ts @@ -27,6 +27,11 @@ const PIECES: Record translate('components.native-chat.failureWords.sendToTryAgain', COPY.sendToTryAgain), + sendAgainToTryOnceMore: () => + translate( + 'components.native-chat.failureWords.sendAgainToTryOnceMore', + COPY.sendAgainToTryOnceMore + ), couldNotStart: (values) => translate('components.native-chat.failureWords.couldNotStart', COPY.couldNotStart, values), couldNotRestart: (values) => diff --git a/src/renderer/src/i18n/en-runtime-required.json b/src/renderer/src/i18n/en-runtime-required.json index 9e4e2b483899..ace6a78c2e5d 100644 --- a/src/renderer/src/i18n/en-runtime-required.json +++ b/src/renderer/src/i18n/en-runtime-required.json @@ -2781,7 +2781,7 @@ "notDelivered": "This message was not delivered.", "notDeliveredSendAgain": "This message was not delivered. Send it again to continue.", "notSignedIn": "{{agent}} is not signed in for the selected account.", - "previousExitUnverifiable": "Orca couldn't stop {{agent}}'s previous process.", + "previousExitUnverifiable": "Couldn't stop {{agent}} from before.", "providerExitedRejection": "{{agent}} stopped before this message was sent.", "providerExitedRow": "{{agent}} stopped while this response was in progress. You can continue in this conversation.", "providerRateLimited": "{{agent}} is rate-limited and retrying.", @@ -2793,6 +2793,7 @@ "queueFull": "Too many messages were waiting for the agent, so this one was not sent.", "runCommandAgain": "Run /{{command}} again.", "sendToTryAgain": "Send your message to try again.", + "sendAgainToTryOnceMore": "Send your message again to try once more.", "signInFirst": "Sign in first.", "signInThenRunCommand": "Sign in, then run /{{command}} again.", "signInThenSend": "Sign in, then send your message again.", diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 6cbf350f6a6c..4a14bbaa6ec3 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -17578,6 +17578,7 @@ "providerStartFailed": "{{agent}} stopped before it finished starting.", "runCommandAgain": "Run /{{command}} again.", "sendToTryAgain": "Send your message to try again.", + "sendAgainToTryOnceMore": "Send your message again to try once more.", "couldNotStart": "{{agent}} couldn't start.", "couldNotRestart": "{{agent}} couldn't restart.", "notSignedIn": "{{agent}} is not signed in for the selected account.", @@ -17631,7 +17632,7 @@ "providerRateLimited": "{{agent}} is rate-limited and retrying.", "providerRetrying": "{{agent}} hit a temporary problem and is retrying.", "providerRetryingQuoted": "{{agent}} is retrying: {{detail}}.", - "previousExitUnverifiable": "Orca couldn't stop {{agent}}'s previous process." + "previousExitUnverifiable": "Couldn't stop {{agent}} from before." }, "writeNotice": { "notDoneReadHistory": "This chat's history couldn't be loaded.", diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 222c0faf8720..73195ae276f9 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -14948,6 +14948,7 @@ "providerStartFailed": "{{agent}} se detuvo antes de terminar de iniciarse.", "runCommandAgain": "Vuelva a ejecutar /{{command}}.", "sendToTryAgain": "Envíe su mensaje para volver a intentarlo.", + "sendAgainToTryOnceMore": "Vuelva a enviar su mensaje para intentarlo de nuevo.", "couldNotStart": "{{agent}} no pudo iniciarse.", "couldNotRestart": "{{agent}} no pudo reiniciarse.", "notSignedIn": "{{agent}} no tiene la sesión iniciada con la cuenta seleccionada.", @@ -15001,7 +15002,7 @@ "providerRateLimited": "{{agent}} alcanzó un límite de solicitudes y está reintentando.", "providerRetrying": "{{agent}} tuvo un problema temporal y está reintentando.", "providerRetryingQuoted": "{{agent}} está reintentando: {{detail}}.", - "previousExitUnverifiable": "Orca no pudo detener el proceso anterior de {{agent}}." + "previousExitUnverifiable": "No se pudo detener la ejecución anterior de {{agent}}." }, "writeNotice": { "notDoneReadHistory": "No se pudo cargar el historial de este chat.", diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index c5aa61e3dd64..63a4898f1dca 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -17417,6 +17417,7 @@ "providerStartFailed": "{{agent}} s'est arrêté avant d'avoir fini de démarrer.", "runCommandAgain": "Relancez /{{command}}.", "sendToTryAgain": "Envoyez votre message pour réessayer.", + "sendAgainToTryOnceMore": "Renvoyez votre message pour réessayer.", "couldNotStart": "{{agent}} n'a pas pu démarrer.", "couldNotRestart": "{{agent}} n'a pas pu redémarrer.", "notSignedIn": "{{agent}} n'est pas connecté avec le compte sélectionné.", @@ -17470,7 +17471,7 @@ "providerRateLimited": "{{agent}} est limité en débit et réessaie.", "providerRetrying": "{{agent}} a rencontré un problème temporaire et réessaie.", "providerRetryingQuoted": "{{agent}} réessaie : {{detail}}.", - "previousExitUnverifiable": "Orca n'a pas pu arrêter le processus précédent de {{agent}}." + "previousExitUnverifiable": "Impossible d'arrêter l'exécution précédente de {{agent}}." }, "writeNotice": { "notDoneReadHistory": "L'historique de ce chat n'a pas pu être chargé.", diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index ce58f420204d..28a9aebac429 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -17354,6 +17354,7 @@ "providerStartFailed": "{{agent}} は起動が完了する前に停止しました。", "runCommandAgain": "/{{command}} をもう一度実行してください。", "sendToTryAgain": "もう一度試すには、メッセージを送信してください。", + "sendAgainToTryOnceMore": "メッセージをもう一度送信して、再度お試しください。", "couldNotStart": "{{agent}} を起動できませんでした。", "couldNotRestart": "{{agent}} を再起動できませんでした。", "notSignedIn": "{{agent}} は選択したアカウントでサインインしていません。", @@ -17407,7 +17408,7 @@ "providerRateLimited": "{{agent}} はレート制限を受けているため、再試行しています。", "providerRetrying": "{{agent}} で一時的な問題が発生したため、再試行しています。", "providerRetryingQuoted": "{{agent}} は再試行しています: {{detail}}。", - "previousExitUnverifiable": "Orca は {{agent}} の前のプロセスを停止できませんでした。" + "previousExitUnverifiable": "以前の {{agent}} を停止できませんでした。" }, "writeNotice": { "notDoneReadHistory": "このチャットの履歴を読み込めませんでした。", diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index bb3afd53c49b..fca3445f027e 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -17354,6 +17354,7 @@ "providerStartFailed": "{{agent}}이(가) 시작을 마치기 전에 중지되었습니다.", "runCommandAgain": "/{{command}}을(를) 다시 실행하세요.", "sendToTryAgain": "다시 시도하려면 메시지를 보내세요.", + "sendAgainToTryOnceMore": "메시지를 다시 보내 한 번 더 시도하세요.", "couldNotStart": "{{agent}}을(를) 시작하지 못했습니다.", "couldNotRestart": "{{agent}}을(를) 다시 시작하지 못했습니다.", "notSignedIn": "{{agent}}이(가) 선택한 계정으로 로그인되어 있지 않습니다.", @@ -17407,7 +17408,7 @@ "providerRateLimited": "{{agent}}이(가) 속도 제한에 걸려 다시 시도하고 있습니다.", "providerRetrying": "{{agent}}에 일시적인 문제가 발생하여 다시 시도하고 있습니다.", "providerRetryingQuoted": "{{agent}}이(가) 다시 시도하고 있습니다: {{detail}}.", - "previousExitUnverifiable": "Orca가 {{agent}}의 이전 프로세스를 중지하지 못했습니다." + "previousExitUnverifiable": "이전에 실행된 {{agent}}을(를) 중지하지 못했습니다." }, "writeNotice": { "notDoneReadHistory": "이 채팅의 기록을 불러오지 못했습니다.", diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index c2d9692fc920..d94177eb848c 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -17371,6 +17371,7 @@ "providerStartFailed": "{{agent}} 在完成启动前已停止。", "runCommandAgain": "请重新运行 /{{command}}。", "sendToTryAgain": "请发送您的消息以重试。", + "sendAgainToTryOnceMore": "请重新发送您的消息，再试一次。", "couldNotStart": "{{agent}} 无法启动。", "couldNotRestart": "{{agent}} 无法重新启动。", "notSignedIn": "{{agent}} 未使用所选账户登录。", @@ -17424,7 +17425,7 @@ "providerRateLimited": "{{agent}} 已被限流，正在重试。", "providerRetrying": "{{agent}} 遇到临时问题，正在重试。", "providerRetryingQuoted": "{{agent}} 正在重试：{{detail}}。", - "previousExitUnverifiable": "Orca 无法停止 {{agent}} 的上一个进程。" + "previousExitUnverifiable": "无法停止之前的 {{agent}}。" }, "writeNotice": { "notDoneReadHistory": "无法加载此聊天的历史记录。", diff --git a/src/shared/agent-session-failure-copy.ts b/src/shared/agent-session-failure-copy.ts index f40ffcfc1d2c..1e555c676d64 100644 --- a/src/shared/agent-session-failure-copy.ts +++ b/src/shared/agent-session-failure-copy.ts @@ -14,6 +14,7 @@ export const AGENT_SESSION_FAILURE_COPY = { providerStartFailed: '{{agent}} stopped before it finished starting.', runCommandAgain: 'Run /{{command}} again.', sendToTryAgain: 'Send your message to try again.', + sendAgainToTryOnceMore: 'Send your message again to try once more.', couldNotStart: "{{agent}} couldn't start.", couldNotRestart: "{{agent}} couldn't restart.", terminalAgentHoldsChat: TERMINAL_AGENT_HOLDS_CHAT, @@ -81,7 +82,7 @@ export const AGENT_SESSION_FAILURE_COPY = { providerRateLimited: '{{agent}} is rate-limited and retrying.', providerRetrying: '{{agent}} hit a temporary problem and is retrying.', providerRetryingQuoted: '{{agent}} is retrying: {{detail}}.', - previousExitUnverifiable: "Orca couldn't stop {{agent}}'s previous process." + previousExitUnverifiable: "Couldn't stop {{agent}} from before." } as const export type AgentSessionFailureCopyId = keyof typeof AGENT_SESSION_FAILURE_COPY diff --git a/src/shared/agent-session-failure-words.test.ts b/src/shared/agent-session-failure-words.test.ts index 98273628214d..4e6b81b265a8 100644 --- a/src/shared/agent-session-failure-words.test.ts +++ b/src/shared/agent-session-failure-words.test.ts @@ -151,6 +151,28 @@ describe('the words written beside a failure fact', () => { ).toBe("Codex couldn't start. Start a new chat to continue.") }) + it('says a start refused beside a process Orca could not stop in its own words', () => { + const refused = (context: { command?: 'compact'; retryControl?: boolean } = {}) => + agentSessionFailureSentence( + { + kind: 'restartFailed', + refusal: { + code: 'agent_session_ownership_unknown', + details: { reason: 'previousExitUnverifiable' } + } + }, + 'rejection', + { agentName: 'Claude', ...context } + ) + expect(refused()).toBe( + "Couldn't stop Claude from before. Send your message again to try once more." + ) + expect(refused({ command: 'compact' })).toBe( + "Couldn't stop Claude from before. Run /compact again." + ) + expect(refused({ retryControl: true })).toBe("Couldn't stop Claude from before.") + }) + it('names the exit a row reports differently from the message it left unsent', () => { expect( agentSessionFailureWords({ kind: 'providerExited' }, { surface: 'row', agentName: 'Claude' }) diff --git a/src/shared/agent-session-failure-words.ts b/src/shared/agent-session-failure-words.ts index 1c9758d81820..74e6e543b533 100644 --- a/src/shared/agent-session-failure-words.ts +++ b/src/shared/agent-session-failure-words.ts @@ -130,12 +130,13 @@ function withRetryCause(sentence: string, cause: string | undefined): string { /** The next step after a start or restart that failed: the command, or the message, again. */ function startRetry( say: AgentSessionFailureSay, - { command, retryControl }: AgentSessionFailureWordsContext + { command, retryControl }: AgentSessionFailureWordsContext, + sendAgain: 'sendToTryAgain' | 'sendAgainToTryOnceMore' = 'sendToTryAgain' ): string[] { if (retryControl) { return [] } - return [command ? say('runCommandAgain', { command }) : say('sendToTryAgain')] + return [command ? say('runCommandAgain', { command }) : say(sendAgain)] } function couldNot(verb: 'couldNotStart' | 'couldNotRestart'): Sentence { @@ -149,7 +150,7 @@ function couldNot(verb: 'couldNotStart' | 'couldNotRestart'): Sentence { if (fact.refusal?.details?.reason === 'previousExitUnverifiable') { return joinSentences([ say('previousExitUnverifiable', agent(say, context)), - ...startRetry(say, context) + ...startRetry(say, context, 'sendAgainToTryOnceMore') ]) } const code = fact.refusal?.code From baf85fcc958c9cf108f9d76bec45c1e69be1c87d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:06:32 -0700 Subject: [PATCH 17/22] fix(native-chat): never drop an agent process end Orca could not confirm Built on the one-exit-handler, close-on-the-child model of #24862. - Claude: an Orca fault whose close cannot prove the child gone reports `end-unproven` instead of staying silent, and the child's later exit (now reported inside that close) settles the exit the adapter kept. - Codex: a forced close that cannot prove the exit reports `end-unproven` for the window before the exit. - Sink failure: an unproven or failing stop reports the end instead of returning silently. - The host keeps such a child on record, closing, with the reported reason, failure and whether work was in flight; every later send, start or Stop joins that close. The exit that ends the record settles as that end (cause exit, the fault's row when work was interrupted); a Stop or close asked for since keeps its own cause and the fault's row. - Dispatch and /compact to a session the adapter no longer holds answer `rejected` (never a throw that reads as delivery unknown), keeping a failed start's wording; Codex says the agent stopped only when its end was seen. - `end-unproven` is handled on its chat's own lane, off the exit chain. --- .../claude-structured-session-adapter.ts | 36 +- ...aude-structured-session-exit-blame.test.ts | 38 ++ ...laude-structured-session-exit-lifecycle.ts | 124 ++++- .../claude/claude-structured-session-state.ts | 6 +- .../codex-structured-journal-translation.ts | 4 + ...ructured-session-adapter-lifecycle.test.ts | 19 +- .../codex-structured-session-adapter.test.ts | 3 +- .../codex/codex-structured-session-adapter.ts | 22 + .../codex/codex-structured-session-state.ts | 6 +- .../codex-structured-session-teardown.ts | 16 +- .../structured-agent-session-adapter.ts | 13 + .../structured-agent-session-child-exit.ts | 39 +- ...session-claude-unproven-fault-send.test.ts | 440 ++++++++++++++++++ ...ession-codex-forced-close-unproven.test.ts | 278 +++++++++++ ...structured-agent-session-event-recovery.ts | 35 +- .../structured-agent-session-host-lifetime.ts | 7 +- .../structured-agent-session-host-types.ts | 11 + ...uctured-agent-session-unproven-end.test.ts | 118 +++++ .../structured-agent-session-unproven-end.ts | 73 +++ ...tured-startup-fault-is-not-an-exit.test.ts | 34 ++ ...d-agent-session-lifecycle-delivery.test.ts | 38 +- ...ctured-agent-session-lifecycle-delivery.ts | 16 +- .../structured-agent-session-runtime.ts | 7 +- .../structured-claude-runtime-adapter.ts | 2 +- .../structured-codex-lifecycle-event.ts | 12 + 25 files changed, 1328 insertions(+), 69 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts create mode 100644 src/main/runtime/structured-codex-lifecycle-event.ts diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index d52177e1d6a6..a1cac172149c 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -26,16 +26,14 @@ import { type ClaudeStructuredSessionAdapterDeps, type ClaudeStructuredSessionEvent } from './claude-structured-session-state' -import { - closeAllClaudeSessions, - closeClaudeSession, - finishClaudeCloseAfterExit -} from './claude-structured-session-close' +import { closeAllClaudeSessions, closeClaudeSession } from './claude-structured-session-close' import { claudeStoppedRequestEndWait } from './claude-request-end-wait' import { drainClaudeObservedExits, + finishClaudeExitedClose, observeClaudeSessionExit, settleClaudeUnexpectedExit, + withClaudeSessionOrRejected, type ClaudeExitLifecycle } from './claude-structured-session-exit-lifecycle' import type { AgentSessionBackgroundTaskState } from '../../shared/agent-session-wire' @@ -64,7 +62,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda private readonly sessions = new Map() private readonly acquisitions = new ClaudeAcquisitionRegistry() private readonly exits = new Map() - private readonly settledExitErrors = new Map() + private readonly settledExits: ClaudeExitLifecycle['settledExits'] = new Map() private readonly exitLifecycle: ClaudeExitLifecycle constructor(private readonly deps: ClaudeStructuredSessionAdapterDeps) { @@ -72,7 +70,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda this.exitLifecycle = { sessions: this.sessions, exits: this.exits, - settledExitErrors: this.settledExitErrors, + settledExits: this.settledExits, deps, emit: (session, event) => this.emit(session, event) } @@ -87,7 +85,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda }) acquire = (input: StructuredAgentSessionAcquireInput): Promise => { - this.settledExitErrors.delete(input.identity.sessionId) + this.settledExits.delete(input.identity.sessionId) return acquireClaudeSession({ input, deps: this.deps, @@ -100,13 +98,9 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda handleExit: (sessionId, attempt, error) => observeClaudeSessionExit(this.exitLifecycle, sessionId, attempt, error), finishClose: (sessionId, attempt) => - finishClaudeCloseAfterExit({ - sessions: this.sessions, - sessionId, - connection: attempt.connection, - deps: this.deps, - afterClose: (close) => this.afterClose(sessionId, close) - }), + finishClaudeExitedClose(this.exitLifecycle, sessionId, attempt.connection, (close) => + this.afterClose(sessionId, close) + ), settleExit: (sessionId, exit) => settleClaudeUnexpectedExit(this.exitLifecycle, sessionId, exit) } @@ -191,10 +185,14 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } dispatch: StructuredAgentSessionAdapter['dispatch'] = (input) => - dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch) + withClaudeSessionOrRejected(this.exitLifecycle, input.sessionId, (session) => + dispatchClaudeTurn(session, input, input.beforeDispatch) + ) compact: NonNullable = (input) => - dispatchClaudeCommand(this.session(input.sessionId), input.command) + withClaudeSessionOrRejected(this.exitLifecycle, input.sessionId, (session) => + dispatchClaudeCommand(session, input.command) + ) cancelTurn: StructuredAgentSessionAdapter['cancelTurn'] = (request) => cancelClaudeStructuredTurn({ @@ -300,7 +298,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda closeSession = (sessionId: string): Promise => // After the close, not before: releasing an exit still settling settles it on the way. - this.closeSessionProcess(sessionId).finally(() => this.settledExitErrors.delete(sessionId)) + this.closeSessionProcess(sessionId).finally(() => this.settledExits.delete(sessionId)) private closeSessionProcess(sessionId: string): Promise { // An exit seen first settles as that exit, whoever asked for the close after it. @@ -335,7 +333,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda // A child that just exited is named by its own diagnostic, not by its absence. throw ( this.exits.get(sessionId)?.error ?? - this.settledExitErrors.get(sessionId) ?? + this.settledExits.get(sessionId)?.error ?? new Error(`no live claude stream-json session for ${sessionId}`) ) } diff --git a/src/main/claude/claude-structured-session-exit-blame.test.ts b/src/main/claude/claude-structured-session-exit-blame.test.ts index 79430131e15c..8f992df7efd0 100644 --- a/src/main/claude/claude-structured-session-exit-blame.test.ts +++ b/src/main/claude/claude-structured-session-exit-blame.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from 'vitest' import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { structuredAgentSessionCommandTurn } from '../native-chat/agent-session-wire/structured-agent-session-command-turn' import type { ClaudeStructuredSessionEvent } from './claude-structured-session-adapter' import { adapterFor, @@ -63,4 +64,41 @@ describe('what a started Claude session says ended it', () => { }) expect(connection.closeCount).toBeGreaterThanOrEqual(1) }) + + it('rejects a message or a compaction for a child it no longer holds, with why it ended', async () => { + const claude = fakeClaude() + const adapter = adapterFor(claude, {}, []) + await adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn-9' }) + const connection = claude.connections[0] + // The close cannot prove the child gone, so the adapter lets go of it and keeps its exit. + connection.close = async () => false + connection.handlers.onExit?.(new Error('claude stream-json exited (code 1): killed')) + await adapter.drainObservedExits() + + const rejected = { + state: 'rejected', + rejection: { kind: 'providerExited' } + } + await expect( + adapter.dispatch({ + sessionId: 'session-1', + clientMessageId: 'client-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 7 + }) + ).resolves.toMatchObject(rejected) + const turn = structuredAgentSessionCommandTurn('client-2') + await expect( + adapter.compact({ + sessionId: 'session-1', + fence: 7, + command: { + clientMessageId: 'client-2', + ...turn, + running: { kind: 'turn', turnId: turn.turnId, state: 'running' } + } + }) + ).resolves.toMatchObject(rejected) + expect(connection.sent).toEqual([]) + }) }) diff --git a/src/main/claude/claude-structured-session-exit-lifecycle.ts b/src/main/claude/claude-structured-session-exit-lifecycle.ts index 854646490a8c..d536a22d71f5 100644 --- a/src/main/claude/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/claude-structured-session-exit-lifecycle.ts @@ -1,11 +1,19 @@ -import { agentSessionFailureFact, providerDiagnosticOf } from '../../shared/agent-session-failure' +import { + agentSessionFailureFact, + providerDiagnosticOf, + type SubmissionRejectionFact +} from '../../shared/agent-session-failure' import { providerExitObserved, providerStartupFailureFact } from '../native-chat/agent-session-wire/structured-agent-session-failure-text' +import type { AgentSessionDispatchOutcome } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import { claudeDispatchRejection } from './claude-structured-dispatch-content' +import type { ClaudeStreamJsonConnection } from './claude-stream-json-connection' import { settledClaudeTurnEndLeaf } from './claude-structured-resume-point' import { claudeRootExitObserved, + finishClaudeCloseAfterExit, settleClaudeExitedSession } from './claude-structured-session-close' import { failClaudeStartup } from './claude-structured-session-startup-state' @@ -17,12 +25,15 @@ import type { ClaudeStructuredSessionEvent } from './claude-structured-session-state' +export type ClaudeSettledExit = { error: Error; failure: SubmissionRejectionFact } + export type ClaudeExitLifecycle = { sessions: Map exits: Map - /** A settled exit's diagnostic, kept for a send admitted before the host heard of the exit. */ - settledExitErrors: Map - deps: Pick + /** A settled exit's diagnostic and why it ended, kept for a send admitted before the host heard + * of the exit. */ + settledExits: Map + deps: Pick emit: (session: ClaudeSession, event: ClaudeStructuredSessionEvent) => void } @@ -34,6 +45,7 @@ export function observeClaudeSessionExit( ): void { const session = lifecycle.sessions.get(sessionId) if (!session || session.connection !== attempt.connection) { + settleRetainedClaudeRootExit(lifecycle, sessionId, attempt.connection) return } lifecycle.sessions.delete(sessionId) @@ -55,6 +67,7 @@ export function observeClaudeSessionExit( .then((proven) => { // A first-hand root exit is final like a proven one: the owner releases the lease on it. if (!proven && !claudeRootExitObserved(session.connection)) { + reportClaudeUnprovenEnd(lifecycle, sessionId, exit) return undefined } return settleClaudeUnexpectedExit(lifecycle, sessionId, exit) @@ -62,6 +75,73 @@ export function observeClaudeSessionExit( .catch(() => undefined) } +/** The root exited after a close began: that close settles it, or one the adapter began after + * letting the session go ends the exit it kept. */ +export function finishClaudeExitedClose( + lifecycle: ClaudeExitLifecycle, + sessionId: string, + connection: ClaudeStreamJsonConnection | null, + afterClose: (close: () => Promise) => Promise +): void { + settleRetainedClaudeRootExit(lifecycle, sessionId, connection) + finishClaudeCloseAfterExit({ + sessions: lifecycle.sessions, + sessionId, + connection, + deps: lifecycle.deps, + afterClose + }) +} + +/** The root of an exit whose close could not prove it gone has now exited: the end it reported + * unproven is final. Only the retained exit's own connection reports it, and only once that + * exit's first close has answered, which publishes the end itself when it saw the root go. */ +function settleRetainedClaudeRootExit( + lifecycle: ClaudeExitLifecycle, + sessionId: string, + connection: ClaudeStreamJsonConnection | null +): void { + const exit = lifecycle.exits.get(sessionId) + if (!exit || exit.connection !== connection) { + return + } + const published = exit.publication ?? Promise.resolve() + exit.publication = published.then(() => + lifecycle.exits.get(sessionId) === exit && claudeRootExitObserved(exit.connection) + ? settleClaudeUnexpectedExit(lifecycle, sessionId, exit).catch(() => undefined) + : undefined + ) +} + +/** Never silence: the host keeps the child on record, closing, until its exit is proven or seen. */ +function reportClaudeUnprovenEnd( + lifecycle: ClaudeExitLifecycle, + sessionId: string, + exit: ClaudeSessionExit +): void { + if (lifecycle.exits.get(sessionId) !== exit) { + return + } + lifecycle.deps.onEvent?.({ + type: 'end-unproven', + sessionId, + reason: exit.error.message, + failure: claudeExitFailure(exit), + fence: exit.session.fence, + acquisitionGeneration: exit.session.acquisitionGeneration + }) +} + +/** A start that never landed says why it failed. After it landed, only the child's own exit blames + * the provider; an Orca fault that closed it is Orca's. */ +function claudeExitFailure(exit: Pick) { + return exit.session.startup.state !== 'proven' + ? providerStartupFailureFact(exit.session.startup.failure ?? exit.error) + : providerExitObserved(exit.error) + ? agentSessionFailureFact('providerExited', { detail: providerDiagnosticOf(exit.error) }) + : agentSessionFailureFact('hostFault') +} + /** Lifecycle recovery is published only after the close ladder ran and proved the tree gone or * observed the root's own exit. */ export function settleClaudeUnexpectedExit( @@ -87,21 +167,12 @@ export function settleClaudeUnexpectedExit( return } exits.delete(sessionId) - lifecycle.settledExitErrors.set(sessionId, exit.error) + lifecycle.settledExits.set(sessionId, { error: exit.error, failure: claudeExitFailure(exit) }) const ended: ClaudeStructuredSessionEvent = { type: 'ended', sessionId, reason: exit.error.message, - // A start that never landed says why it failed. After it landed, only the child's own exit - // blames the provider; an Orca fault that closed it is Orca's. - failure: - exit.session.startup.state !== 'proven' - ? providerStartupFailureFact(exit.session.startup.failure ?? exit.error) - : providerExitObserved(exit.error) - ? agentSessionFailureFact('providerExited', { - detail: providerDiagnosticOf(exit.error) - }) - : agentSessionFailureFact('hostFault'), + failure: claudeExitFailure(exit), cause: 'unexpected-exit', fence: exit.session.fence, acquisitionGeneration: exit.session.acquisitionGeneration, @@ -117,6 +188,29 @@ export function settleClaudeUnexpectedExit( return exit.settlementPromise } +/** A message for a child this adapter no longer serves was never written, so it is rejected with + * why that child ended, never left in doubt. */ +function rejectClaudeDetachedDispatch( + lifecycle: Pick, + sessionId: string +): AgentSessionDispatchOutcome { + const exit = lifecycle.exits.get(sessionId) + const failure = exit + ? claudeExitFailure(exit) + : (lifecycle.settledExits.get(sessionId)?.failure ?? agentSessionFailureFact('providerExited')) + return { state: 'rejected', ...claudeDispatchRejection(failure) } +} + +/** Runs `run` on the live session, or rejects what was meant for a child this adapter let go of. */ +export async function withClaudeSessionOrRejected( + lifecycle: Pick, + sessionId: string, + run: (session: ClaudeSession) => Promise +): Promise { + const session = lifecycle.sessions.get(sessionId) + return session ? run(session) : rejectClaudeDetachedDispatch(lifecycle, sessionId) +} + /** Wait for each first-hand exit's publication, including exits observed while waiting. */ export async function drainClaudeObservedExits( exits: Map diff --git a/src/main/claude/claude-structured-session-state.ts b/src/main/claude/claude-structured-session-state.ts index 876090c07e4d..6c8287a0c8fe 100644 --- a/src/main/claude/claude-structured-session-state.ts +++ b/src/main/claude/claude-structured-session-state.ts @@ -6,7 +6,10 @@ import type { AgentSessionJournalIdentity } from '../../shared/agent-session-journal-types' import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' -import type { StructuredAgentSessionStartedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { + StructuredAgentSessionEndUnprovenEvent, + StructuredAgentSessionStartedEvent +} from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { ClaudeStreamJsonConnection, openClaudeStreamJsonConnection @@ -67,6 +70,7 @@ export type ClaudeStructuredSessionEvent = | { type: 'auth-diagnostic'; sessionId: string; diagnostic: ClaudeAuthDiagnostic } /** Startup facts applied and saved options restored; held prompts are about to be written. */ | StructuredAgentSessionStartedEvent + | StructuredAgentSessionEndUnprovenEvent | { type: 'ended' sessionId: string diff --git a/src/main/codex/codex-structured-journal-translation.ts b/src/main/codex/codex-structured-journal-translation.ts index 20fd8bf58df3..d154eb717066 100644 --- a/src/main/codex/codex-structured-journal-translation.ts +++ b/src/main/codex/codex-structured-journal-translation.ts @@ -140,6 +140,10 @@ export function createCodexJournalTranslator( }) }, handle: (event) => { + // Host lifecycle only: the journal records this child's end once its exit is seen. + if (event.type === 'end-unproven') { + return CODEX_JOURNAL_ADMITTED + } if (event.type === 'ended') { const streamAdmission = flushStreams() if (!streamAdmission.accepted) { diff --git a/src/main/codex/codex-structured-session-adapter-lifecycle.test.ts b/src/main/codex/codex-structured-session-adapter-lifecycle.test.ts index e3211f320e85..269b141a96f1 100644 --- a/src/main/codex/codex-structured-session-adapter-lifecycle.test.ts +++ b/src/main/codex/codex-structured-session-adapter-lifecycle.test.ts @@ -187,7 +187,8 @@ describe('CodexStructuredSessionAdapter lifecycle', () => { body: USER_MESSAGE, fence: 7 }) - ).rejects.toThrow('no live codex app-server') + // Never written to a child it no longer serves: rejected, never left in doubt. + ).resolves.toMatchObject({ state: 'rejected', rejection: { kind: 'providerExited' } }) expect(await adapter.historyFilePath({ identity: identityFor('session-1') })).toBe( '/rollouts/abc.jsonl' ) @@ -195,6 +196,22 @@ describe('CodexStructuredSessionAdapter lifecycle', () => { expect(events.filter((event) => event.type === 'ended')).toHaveLength(1) }) + it('rejects a message for a connection that broke while the child may still run, without saying Codex stopped', async () => { + const codex = fakeCodex() + const adapter = await acquired(codex) + codex.connections[0].closed = true + + await expect( + adapter.dispatch({ + sessionId: 'session-1', + clientMessageId: 'client-1', + body: USER_MESSAGE, + fence: 7 + }) + ).resolves.toMatchObject({ state: 'rejected', rejection: { kind: 'writeFailed' } }) + expect(codex.connections[0].calls.some((call) => call.method === 'turn/start')).toBe(false) + }) + it('keeps the live session when a child it already replaced dies', async () => { const codex = fakeCodex() const events: CodexStructuredSessionEvent[] = [] diff --git a/src/main/codex/codex-structured-session-adapter.test.ts b/src/main/codex/codex-structured-session-adapter.test.ts index 70b2ecd7147a..23af8fec5db4 100644 --- a/src/main/codex/codex-structured-session-adapter.test.ts +++ b/src/main/codex/codex-structured-session-adapter.test.ts @@ -265,7 +265,8 @@ describe('CodexStructuredSessionAdapter.acquire', () => { body: USER_MESSAGE, fence: 7 }) - ).rejects.toThrow('no live codex app-server') + // Never written to a child it no longer serves: rejected, never left in doubt. + ).resolves.toMatchObject({ state: 'rejected', rejection: { kind: 'providerExited' } }) }) it('classifies launch validation failure as pre-spawn without opening a child', async () => { diff --git a/src/main/codex/codex-structured-session-adapter.ts b/src/main/codex/codex-structured-session-adapter.ts index 2d3e1eda8fe1..96015983ba5f 100644 --- a/src/main/codex/codex-structured-session-adapter.ts +++ b/src/main/codex/codex-structured-session-adapter.ts @@ -200,6 +200,10 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap requestedAt?: number beforeDispatch?: () => Promise }): Promise { + const detached = this.detachedRejection(input.sessionId) + if (detached) { + return detached + } const session = this.session(input.sessionId) session.dispatchPending = true try { @@ -210,6 +214,20 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap } } + /** A closed connection writes nothing, so what was meant for it was never sent: rejected, not + * doubt. Only a seen end says Codex stopped; a closing or broken connection may still have it. */ + private detachedRejection(sessionId: string): AgentSessionDispatchOutcome | null { + const session = this.sessions.get(sessionId) + if (session && !session.ended && !session.connection.closed) { + return null + } + const gone = !session || session.ended + return { + state: 'rejected', + ...codexDispatchRejection(agentSessionFailureFact(gone ? 'providerExited' : 'writeFailed')) + } + } + cancelTurn: StructuredAgentSessionAdapter['cancelTurn'] = (request) => cancelCodexStructuredTurn({ request, @@ -230,6 +248,10 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap /** The ack is Codex's receipt; the translator ends the command's turn from the turn it opens. */ compact: NonNullable = async (input) => { + const detached = this.detachedRejection(input.sessionId) + if (detached) { + return detached + } const session = this.session(input.sessionId) session.translator?.beginCommand(input.command) try { diff --git a/src/main/codex/codex-structured-session-state.ts b/src/main/codex/codex-structured-session-state.ts index 78e7291fd742..fed1920e3adf 100644 --- a/src/main/codex/codex-structured-session-state.ts +++ b/src/main/codex/codex-structured-session-state.ts @@ -19,7 +19,10 @@ import type { CodexDispatchEchoes } from './codex-structured-dispatch-echo' import type { AgentChildWorkEvidence } from '../../shared/agent-status-child-work-evidence' import type { CodexBackgroundTaskTracker } from './codex-background-task-tracker' import type { CodexJournalTranslator } from './codex-structured-journal-translation' -import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' +import type { + StructuredAgentSessionEndedEvent, + StructuredAgentSessionEndUnprovenEvent +} from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { CodexStructuredPermissionPolicy } from './codex-structured-permission-policy' import type { AgentModelCatalogSessionAccess, @@ -68,6 +71,7 @@ export type CodexStructuredSessionEvent = promptKey: string } | StructuredAgentSessionEndedEvent + | StructuredAgentSessionEndUnprovenEvent /** Translator-only compatibility for callers that do not participate in host recovery. */ | { type: 'ended'; sessionId: string; reason: string; observedAt?: number } diff --git a/src/main/codex/codex-structured-session-teardown.ts b/src/main/codex/codex-structured-session-teardown.ts index e7395fb3c845..de427305c092 100644 --- a/src/main/codex/codex-structured-session-teardown.ts +++ b/src/main/codex/codex-structured-session-teardown.ts @@ -4,6 +4,7 @@ // session owns are cleared exactly once, and only when the child was actually // proven stopped — a refused close leaves the session indexed for a retry. +import { agentSessionFailureFact } from '../../shared/agent-session-failure' import { AgentSessionAcquisitionRootExitObservedError } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' import { @@ -81,7 +82,20 @@ export class CodexStructuredSessionTeardown { expectedFence: fence, expectedAcquisitionGeneration: acquisitionGeneration, unexpectedReason: reason - }).then((closed) => this.settled(sessionId, closed)) + }).then((closed) => { + // Never silence: the host keeps the child, closing, until its exit is seen or proven. + if (!closed && this.deps.sessions.get(sessionId) === session && !session.ended) { + this.deps.onEvent?.({ + type: 'end-unproven', + sessionId, + reason: reason.message, + failure: agentSessionFailureFact('hostFault'), + fence, + acquisitionGeneration + }) + } + return this.settled(sessionId, closed) + }) } closeAll = (): Promise => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index 318bada1fe52..604d2b96c59b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -212,8 +212,21 @@ export type StructuredAgentSessionStartedEvent = { restoreSkippedOptions: readonly string[] } +/** The adapter no longer serves this child, and its stop could not prove the root gone: it may + * still be running. The adapter keeps it for a retry, and its exit, whenever seen, is `ended`. */ +export type StructuredAgentSessionEndUnprovenEvent = { + type: 'end-unproven' + sessionId: string + /** Log text only. */ + reason: string + failure?: SubmissionRejectionFact + fence: number + acquisitionGeneration: string +} + export type StructuredAgentSessionLifecycleEvent = | StructuredAgentSessionEndedEvent + | StructuredAgentSessionEndUnprovenEvent | StructuredAgentSessionStartedEvent /** Whether the provider child behind an acquisition has proven its start. A publish-first diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 2980030c2b6b..6c90e549655d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -8,6 +8,7 @@ import { structuredAgentSessionFailureWordsContext } from './structured-agent-se import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' import type { + StructuredAgentSessionEndedChild, StructuredAgentSessionHostSession, StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' @@ -98,8 +99,11 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< if (!session || session.child !== child) { return } - const { expected } = exit const close = child.close + // An end the adapter reported but could not prove is settled as that end, however its exit came. + const reported = close?.reported + const expected = exit.expected && !reported + const failure = reported ? reported.failure : exit.failure // Receipt of the exit is the one end time the host may record for a running turn. const observedAt = exit.observedAt ?? context.now() // The host's own phase decides, so a provider that omits the flag still gets a start that @@ -109,14 +113,15 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< endProviderChild(session, { generation: child.generation, fence: child.fence, - // A close keeps the cause of the stop that asked for it, and ends where it was asked. - cause: expected ? (close?.cause ?? 'evict') : 'exit', - reason: expected ? (close?.reason ?? null) : exit.reason, - ...(!expected && exit.failure ? { failure: exit.failure } : {}), + // A close keeps the cause of the stop that asked for it, and ends where it was asked. A + // reported end keeps a Stop or close asked since; none asked, it was the child's own end. + cause: expected ? (close?.cause ?? 'evict') : reported ? reportedEndCause(close) : 'exit', + reason: expected ? (close?.reason ?? null) : (reported?.reason ?? exit.reason), + ...(!expected && failure ? { failure } : {}), duringStartup: exitedDuringStartup, // The adapter publishes an exit only once it saw the root go, first-hand or proven. rootGone: true, - ...(expected && close ? { endedAt: close.requestedAt } : {}) + ...((expected || reported) && close ? { endedAt: close.requestedAt } : {}) }) context.publishStatus?.(sessionId) } @@ -157,12 +162,14 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< showUnexpectedExitOutcome: !expected && (exitedDuringStartup || - unfinishedStructuredAgentSessionWorkWasInterrupted( - unfinishedWork, - session.journal, - observedAt - )), - ...(!expected && exit.failure ? { exitFailure: exit.failure } : {}), + (reported + ? reported.interruptedWork + : unfinishedStructuredAgentSessionWorkWasInterrupted( + unfinishedWork, + session.journal, + observedAt + ))), + ...(!expected && failure ? { exitFailure: failure } : {}), ...(!expected && exitedDuringStartup && child.generation ? { exitedDuringStartup: { generation: child.generation } } : {}) @@ -208,6 +215,14 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< } } +/** A Stop, close or quit asked for since an unproven end keeps its cause; with none, the end was + * the child's own. */ +function reportedEndCause( + close: StructuredAgentSessionProviderChild['close'] +): StructuredAgentSessionEndedChild['cause'] { + return close && close.cause !== 'host-stop' ? close.cause : 'exit' +} + function logExitFailure( context: Pick, sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts new file mode 100644 index 000000000000..c7621021bb4e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts @@ -0,0 +1,440 @@ +// A Claude child ended by an Orca fault whose close could not prove it gone, then the user's next +// messages, on the shipping adapter and host. The child may still be running, so the host keeps it +// on record, closing: a message joins that close and retries the kill, and is refused readably while +// the exit stays unproven, never reaching the old child and never written off as `unknown`. The old +// child's exit ends the record as the fault it was, and the next message goes to a new child. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentStatusStructuredSessionSubject } from '../../../shared/agent-status-subject' +import { AgentHookServer } from '../../agent-hooks/server' +import { ClaudeStructuredSessionAdapter } from '../../claude/claude-structured-session-adapter' +import { + fakeClaude, + PROVIDER_SESSION_ID, + type FakeConnection +} from '../../claude/claude-structured-session-test-support' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } +const CAPABILITIES = ['interrupt_receipt_v1', 'interrupt_cancel_queued_v1', 'msg_lifecycle_v1'] + +let root: string +let host: StructuredAgentSessionHost +let adapter: ClaudeStructuredSessionAdapter +let store: AgentSessionRecordStore +let claude: ReturnType +let server: AgentHookServer +let statusSubject: AgentStatusStructuredSessionSubject | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-claude-unproven-fault-send-')) + resetHostTestOperationIds() + server = new AgentHookServer() + statusSubject = undefined + claude = fakeClaude({ + replayUuid: null, + routes: { interrupt: () => ({ still_queued: [], cancelled: [] }) } + }) + const lifecycle: Promise[] = [] + adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: root, + claudeConfigDir: join(root, 'claude-home'), + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + resumesTranscript: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0, + continuesChain: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0 + }), + onEvent: (event) => { + const mapped = structuredClaudeLifecycleEvent(event) + if (mapped) { + lifecycle.push(host.handleAdapterEvent(mapped)) + } + }, + onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), + onChildWorkEvidence: (sessionId, evidence) => + host.publishChildWorkEvidence(sessionId, evidence), + openConnection: claude.openConnection, + readProcessStartTime: async () => 1_700_000_000_000, + now: () => NOW + }) + store = await openTestAgentSessionRecordStore(root) + host = new StructuredAgentSessionHost({ + store, + adapter: Object.assign(adapter, { supportsCreate: () => true }), + journalDatabase: openTestJournalHostDatabase(root), + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + logger: recordingStructuredAgentSessionLogger().logger, + idleSweep: { intervalMs: 1_000_000_000, idleMs: 0 }, + statusSink: { + publish: (summary, subject) => { + statusSubject = subject + server.ingestStructuredStatus(summary, subject) + }, + forget: (subject) => server.dropStructuredStatus(subject), + publishChildWork: (subject, evidence, provider) => + server.ingestStructuredChildWork(subject, evidence, provider), + readChildWork: (subject) => server.getStructuredChildWorkViews(subject) + }, + now: () => NOW + }) + const params = hostTestAttachParams(null, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION_ID, leafUuid: null } + }) + expect(await host.attach(CALLER, params)).toMatchObject({ ok: true }) + await adapter.awaitStarted(SESSION) + await Promise.all(lifecycle) +}) + +afterEach(async () => { + // Every child the test left unproven has exited by now. + for (const connection of claude.connections) { + connection.exitVerdict = { root: 'exited', tree: 'exited' } + } + await adapter.closeAll() + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function eventually(assertion: () => T | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +function frame(connection: FakeConnection, message: Record): void { + connection.handlers.onMessage?.({ session_id: PROVIDER_SESSION_ID, ...message }) +} + +function wrote(connection: FakeConnection, text: string): boolean { + return connection.sent.some((message) => JSON.stringify(message).includes(text)) +} + +function envelope( + method: 'agentSession.send' | 'agentSession.cancel', + fields: Parameters[0]['fields'] +) { + return { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: store.getRecord(SESSION)!.lease.runtimeFence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method, + sessionId: SESSION, + fields + }) + } +} + +async function send(text: string): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + if (!sent.ok) { + throw new Error(`send refused: ${JSON.stringify(sent.refusal)}`) + } + return sent.value.clientMessageId +} + +function stop() { + return host.cancel(CALLER, { envelope: envelope('agentSession.cancel', {}) }) +} + +async function submission(clientMessageId: string) { + await host.flushStreamedEvents(SESSION) + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) +} + +function laneDrained(): Promise { + return host['tasks'].serialize(SESSION, async () => {}) +} + +function hostSession() { + return host['sessions'].get(SESSION) +} + +function liveChildRecords() { + const records = statusSubject ? server.getStructuredChildWorkViews(statusSubject) : [] + return records.filter((record) => record.membership === 'live') +} + +/** The lead's turn ends with a background task still running, as a real Claude leaves it. */ +async function turnWithBackgroundTask(connection: FakeConnection): Promise { + const text = 'Start a background agent.' + await send(text) + await eventually(() => expect(wrote(connection, text)).toBe(true)) + frame(connection, { + type: 'system', + subtype: 'init', + uuid: 'init-1', + model: 'claude-sonnet-5', + capabilities: CAPABILITIES + }) + const written = connection.sent.at(-1)! + frame(connection, { ...written, uuid: written.uuid }) + frame(connection, { + type: 'assistant', + uuid: 'leaf-1', + parent_tool_use_id: null, + message: { id: 'msg-1', role: 'assistant', content: [{ type: 'text', text: 'Started.' }] } + }) + frame(connection, { + type: 'system', + subtype: 'task_started', + uuid: 'task-start', + task_id: 'background-1', + task_type: 'local_agent', + is_backgrounded: true, + description: 'bg agent' + }) + frame(connection, { type: 'result', subtype: 'success', is_error: false, uuid: 'result-1' }) + await host.flushStreamedEvents(SESSION) +} + +/** As a loaded machine leaves it: no close can prove the child gone; it still runs. */ +function closesUnproven(connection: FakeConnection, pending?: Promise): void { + connection.exitVerdict = { root: 'live', tree: 'unverifiable' } + connection.close = async () => { + connection.closeCount += 1 + connection.closed = true + if (connection.closeCount > 1) { + await pending + } + return connection.exitVerdict.root === 'exited' + } +} + +/** The journal refuses a background-task row, and the adapter's own fault path ends the child. */ +async function faultTheJournal(connection: FakeConnection): Promise { + const sink = host['runtimeState'].eventSinkFor(SESSION).sink + vi.spyOn(sink, 'tryAppendResolvedItemAndPublish').mockReturnValueOnce({ + accepted: false, + reason: 'failed' + }) + frame(connection, { + type: 'system', + subtype: 'task_started', + uuid: 'task-start-2', + task_id: 'background-2', + task_type: 'local_bash', + is_backgrounded: true, + description: 'npm run build' + }) +} + +/** The old child's own exit, as the real connection reports it: inside the close it began. */ +function exits(connection: FakeConnection): void { + connection.exitVerdict = { root: 'exited', tree: 'exited' } + connection.handlers.onExit?.(new Error('claude stream-json exited (code 0)'), { expected: true }) +} + +/** The fault ended the child, its close could not prove it, and the host was told. */ +async function faultedUnproven(connection: FakeConnection, pending?: Promise): Promise { + closesUnproven(connection, pending) + await faultTheJournal(connection) + await adapter.drainObservedExits() + await laneDrained() +} + +const REFUSED = "Couldn't stop Claude from before." +const FAULT_ROW = "Orca ran into a problem, so this didn't go through. Try again." + +async function faultRows() { + await host.flushStreamedEvents(SESSION) + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' && item.body.failure?.kind === 'hostFault' ? [item.body.text] : [] + ) +} + +async function settled(clientMessageId: string) { + return eventually(async () => { + const entry = await submission(clientMessageId) + expect(entry?.dispatchState).not.toBe('pending') + return entry + }) +} + +it('refuses messages readably after a journal fault whose close was unproven, never sending them or leaving them unknown, and sends the next once the old child exits', async () => { + const connection = claude.connections[0]! + await turnWithBackgroundTask(connection) + await faultedUnproven(connection) + const sentBefore = connection.sent.length + + // Before: the dispatch threw on a child the adapter no longer held, and the send read `unknown`. + const held = await send('Hello again after the fault.') + expect(await settled(held)).toMatchObject({ dispatchState: 'rejected' }) + expect((await submission(held))?.reason).toContain(REFUSED) + expect(connection.sent.length).toBe(sentBefore) + // The child may still run: the host keeps it on record, closing, with the fault it reported. + expect(hostSession()?.child).toMatchObject({ + phase: 'ready', + close: { cause: 'host-stop', reported: { failure: { kind: 'hostFault' } } } + }) + const closes = connection.closeCount + + // Stop at rest joins the close and tries the kill again. + await expect(stop()).resolves.toMatchObject({ ok: true }) + expect(connection.closeCount).toBe(closes + 1) + expect(hostSession()?.child?.close).toBeDefined() + + // The old child exits: the record ends with the fault, with no sweep tick, and work goes on. The + // Stop asked for since is the user's. + exits(connection) + await eventually(() => expect(hostSession()?.child).toBeNull()) + expect(hostSession()?.lastEndedChild).toMatchObject({ + cause: 'user-stop', + failure: { kind: 'hostFault' } + }) + expect(await faultRows()).toEqual([FAULT_ROW]) + expect(liveChildRecords().map((record) => record.providerId)).not.toContain('background-1') + + const next = await send('Carry on.') + const resumed = await eventually(() => { + const started = claude.connections.at(-1)! + expect(started).not.toBe(connection) + expect(wrote(started, 'Carry on.')).toBe(true) + return started + }) + const echoed = resumed.sent.at(-1)! + frame(resumed, { ...echoed, uuid: echoed.uuid }) + await eventually(async () => expect((await submission(next))?.dispatchState).toBe('accepted')) + expect(wrote(connection, 'Carry on.')).toBe(false) + + await expect(host['lifetime'].idleSweep.tick()).resolves.toBeUndefined() +}) + +it('rejects, never leaves in doubt, a message dispatched after the adapter let go of a child its close has not yet answered for', async () => { + const connection = claude.connections[0]! + await turnWithBackgroundTask(connection) + const closing = Promise.withResolvers() + closesUnproven(connection, closing.promise) + await faultTheJournal(connection) + // The fault's first close gave up; the exit path's own close is still running. + await eventually(() => expect(connection.closeCount).toBe(2)) + const sentBefore = connection.sent.length + + const raced = await send('Sent while the close runs.') + expect(await settled(raced)).toMatchObject({ + dispatchState: 'rejected', + rejection: { kind: 'hostFault' } + }) + expect(connection.sent.length).toBe(sentBefore) + + closing.resolve() + await adapter.drainObservedExits() + await eventually(() => expect(hostSession()?.child?.close?.reported).toBeDefined()) +}) + +it('keeps the child of a journal sink failure whose stop was unproven on record, closing, with the fault', async () => { + const connection = claude.connections[0]! + await turnWithBackgroundTask(connection) + closesUnproven(connection) + // The host's own recovery: the journal write fails for good, and Orca stops the provider. + const journal = hostSession()!.journal + vi.spyOn(journal, 'appendItem').mockRejectedValueOnce(new Error('disk unavailable')) + frame(connection, { + type: 'assistant', + uuid: 'leaf-2', + parent_tool_use_id: null, + message: { id: 'msg-2', role: 'assistant', content: [{ type: 'text', text: 'Lost.' }] } + }) + + await eventually(() => + expect(hostSession()?.child?.close).toMatchObject({ + cause: 'host-stop', + reported: { failure: { kind: 'hostFault' } } + }) + ) + expect(connection.closeCount).toBeGreaterThan(0) + host['runtimeState'].eventSinkFor(SESSION) +}) + +/** A turn still running when the fault comes. */ +async function turnRunning(connection: FakeConnection): Promise { + const text = 'Do a long thing.' + await send(text) + await eventually(() => expect(wrote(connection, text)).toBe(true)) + frame(connection, { + type: 'system', + subtype: 'init', + uuid: 'init-1', + model: 'claude-sonnet-5', + capabilities: CAPABILITIES + }) + const written = connection.sent.at(-1)! + frame(connection, { ...written, uuid: written.uuid }) + frame(connection, { + type: 'assistant', + uuid: 'leaf-1', + parent_tool_use_id: null, + message: { id: 'msg-1', role: 'assistant', content: [{ type: 'text', text: 'Working...' }] } + }) + await host.flushStreamedEvents(SESSION) +} + +it.each([ + ['proven at once', false], + ['proven only by the late exit', true] +])('tells why a turn a journal fault cut short stopped, its close %s', async (_name, unproven) => { + const connection = claude.connections[0]! + await turnRunning(connection) + if (unproven) { + await faultedUnproven(connection) + expect(await faultRows()).toEqual([]) + exits(connection) + } else { + connection.exitVerdict = { root: 'exited', tree: 'exited' } + await faultTheJournal(connection) + await adapter.drainObservedExits() + } + await eventually(() => expect(hostSession()?.child).toBeNull()) + + // The same end reads the same however long its proof took. + expect(await faultRows()).toEqual([FAULT_ROW]) + expect(hostSession()?.lastEndedChild).toMatchObject({ + cause: 'exit', + failure: { kind: 'hostFault' } + }) +}) + +it("still tells why a turn a fault cut short stopped when the user's Stop comes before the late exit", async () => { + const connection = claude.connections[0]! + await turnRunning(connection) + await faultedUnproven(connection) + expect(hostSession()?.child?.close?.reported).toBeDefined() + + // The turn still reads running, so the user presses Stop; its close cannot prove the exit either. + await expect(stop()).resolves.toMatchObject({ ok: true }) + await laneDrained() + exits(connection) + await eventually(() => expect(hostSession()?.child).toBeNull()) + + expect(await faultRows()).toEqual([FAULT_ROW]) + // The Stop stays the user's; the fault that came first says why the turn ended. + expect(hostSession()?.lastEndedChild).toMatchObject({ + cause: 'user-stop', + failure: { kind: 'hostFault' } + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts new file mode 100644 index 000000000000..a402192d7a1d --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts @@ -0,0 +1,278 @@ +// A Codex app-server Orca force-closed after a frame it could not record, whose close could not prove +// it gone, on the shipping adapter and host. The child may still be running, so the host keeps it on +// record, closing: a message sent meanwhile joins that close and is refused readably, never left in +// doubt, and the old app-server's exit ends the record as the fault it was. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { agentSessionProviderHandleChainHead } from '../../../shared/agent-session-provider-handle' +import { fakeCodex } from '../../codex/codex-structured-session-adapter-fixture' +import { + CodexStructuredSessionAdapter, + type CodexStructuredSessionEvent +} from '../../codex/codex-structured-session-adapter' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { structuredCodexLifecycleEvent } from '../../runtime/structured-codex-lifecycle-event' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +let root: string +let host: StructuredAgentSessionHost +let adapter: CodexStructuredSessionAdapter +let store: AgentSessionRecordStore +let codex: ReturnType +let events: CodexStructuredSessionEvent[] +const unexited: (() => void)[] = [] + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-codex-forced-close-unproven-')) + resetHostTestOperationIds() + events = [] + codex = fakeCodex() + let generation = 0 + adapter = new CodexStructuredSessionAdapter({ + // As the shipping resolver: a restart resumes the thread the record names. + resolveLaunch: async () => { + const head = agentSessionProviderHandleChainHead( + store.getRecord(SESSION)?.providerHandleChain ?? [] + ) + return { + command: 'codex', + args: ['app-server'], + cwd: root, + codexHome: null, + resumeThreadId: head?.handle.provider === 'codex' ? head.handle.threadId : null + } + }, + // As the runtime routes them. + onEvent: (event) => { + events.push(event) + const lifecycleEvent = structuredCodexLifecycleEvent(event) + if (lifecycleEvent) { + void host.handleAdapterEvent(lifecycleEvent) + } + }, + openConnection: codex.openConnection, + readProcessStartTime: async () => 1_700_000_000_000, + now: () => NOW, + mintAcquisitionGeneration: () => `generation-${++generation}` + }) + store = await openTestAgentSessionRecordStore(root) + host = new StructuredAgentSessionHost({ + store, + adapter: Object.assign(adapter, { supportsCreate: () => true }), + journalDatabase: openTestJournalHostDatabase(root), + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-1', + logger: recordingStructuredAgentSessionLogger().logger, + now: () => NOW + }) + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) +}) + +afterEach(async () => { + // Every app-server a test left running has exited by now. + for (const exit of unexited.splice(0)) { + exit() + } + await adapter.closeAll() + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +type CodexConnection = (typeof codex.connections)[number] + +async function send(text: string): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: store.getRecord(SESSION)!.lease.runtimeFence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + if (!sent.ok) { + throw new Error(`send refused: ${JSON.stringify(sent.refusal)}`) + } + return sent.value.clientMessageId +} + +async function submission(clientMessageId: string) { + await host.flushStreamedEvents(SESSION) + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) +} + +function laneDrained(): Promise { + return host['tasks'].serialize(SESSION, async () => {}) +} + +function hostSession() { + return host['sessions'].get(SESSION) +} + +function startedTurnWith(connection: CodexConnection, text: string): boolean { + return connection.calls.some( + (call) => call.method === 'turn/start' && JSON.stringify(call.params).includes(text) + ) +} + +/** As the real connection: a close proves nothing until the app-server's exit is observed. */ +function closesOnlyOnceExited(connection: CodexConnection): { exit: () => void } { + let exited = false + connection.close = async () => { + connection.closeCount += 1 + connection.closed = true + return exited + } + // As the real connection reports it: inside the close Orca began. + const exit = (): void => { + if (!exited) { + exited = true + connection.handlers.onExit?.(new Error('codex app-server exited'), { expected: true }) + } + } + unexited.push(exit) + return { exit } +} + +async function resumedWith(connection: CodexConnection, text: string): Promise { + return vi.waitFor(() => { + const next = codex.connections.at(-1)! + expect(next).not.toBe(connection) + expect(startedTurnWith(next, text)).toBe(true) + return next + }) +} + +const FAULT_ROW = "Orca ran into a problem, so this didn't go through. Try again." + +async function faultRows() { + await host.flushStreamedEvents(SESSION) + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' && item.body.failure?.kind === 'hostFault' ? [item.body.text] : [] + ) +} + +/** A frame the journal cannot record: Orca force-closes the app-server as an unexpected end. */ +function forceCloseOnAnUnrecordableFrame(connection: CodexConnection): void { + const sink = host['runtimeState'].eventSinkFor(SESSION).sink + vi.spyOn(sink, 'tryAppendItem').mockReturnValueOnce({ accepted: false, reason: 'failed' }) + connection.handlers.onUnhandledFrame?.('frame:unknown-method', { method: 'mystery/event' }) +} + +function turnRunning(connection: CodexConnection): Promise { + const threadId = adapter['sessions'].get(SESSION)!.threadId + connection.handlers.onNotification?.('turn/started', { threadId, turn: { id: 'turn-1' } }) + return host.flushStreamedEvents(SESSION) +} + +it('keeps an app-server Orca force-closed unproven on record, refuses the next message readably, and sends the one after its exit', async () => { + const connection = codex.connections[0]! + const old = closesOnlyOnceExited(connection) + forceCloseOnAnUnrecordableFrame(connection) + + await vi.waitFor(() => expect(events.some((event) => event.type === 'end-unproven')).toBe(true)) + await laneDrained() + expect(hostSession()?.child).toMatchObject({ + generation: 'generation-1', + close: { cause: 'host-stop', reported: { failure: { kind: 'hostFault' } } } + }) + + const held = await send('Carry on.') + await vi.waitFor(async () => expect((await submission(held))?.dispatchState).toBe('rejected')) + expect((await submission(held))?.reason).toContain("Couldn't stop Codex from before.") + expect(startedTurnWith(connection, 'Carry on.')).toBe(false) + expect(codex.connections).toHaveLength(1) + + old.exit() + await vi.waitFor(() => expect(hostSession()?.child).toBeNull()) + await send('Go on.') + const resumed = await resumedWith(connection, 'Go on.') + + expect(startedTurnWith(connection, 'Go on.')).toBe(false) + expect(resumed.closeCount).toBe(0) +}) + +it("drops an exit from an app-server that is no longer the chat's", async () => { + const connection = codex.connections[0]! + const old = closesOnlyOnceExited(connection) + forceCloseOnAnUnrecordableFrame(connection) + await vi.waitFor(() => expect(hostSession()?.child?.close?.reported).toBeDefined()) + old.exit() + await vi.waitFor(() => expect(hostSession()?.child).toBeNull()) + await send('Carry on.') + const resumed = await resumedWith(connection, 'Carry on.') + const reports = events.filter((event) => event.type === 'ended').length + + connection.handlers.onExit?.(new Error('codex app-server exited again'), { expected: true }) + await laneDrained() + + expect(events.filter((event) => event.type === 'ended')).toHaveLength(reports) + expect(resumed.closeCount).toBe(0) + expect(hostSession()?.child?.generation).toBe('generation-2') +}) + +it.each([ + ['proven at once', false], + ['proven only by the late exit', true] +])('tells why a turn a forced close cut short stopped, its close %s', async (_name, unproven) => { + const connection = codex.connections[0]! + const old = unproven ? closesOnlyOnceExited(connection) : null + await turnRunning(connection) + forceCloseOnAnUnrecordableFrame(connection) + if (old) { + await vi.waitFor(() => expect(hostSession()?.child?.close?.reported).toBeDefined()) + expect(await faultRows()).toEqual([]) + old.exit() + } + await vi.waitFor(() => expect(hostSession()?.child).toBeNull()) + + // The same end reads the same however long its proof took. + await vi.waitFor(async () => expect(await faultRows()).toEqual([FAULT_ROW])) + expect(hostSession()?.lastEndedChild).toMatchObject({ + cause: 'exit', + failure: { kind: 'hostFault' } + }) +}) + +it('still tells why a turn a forced close cut short stopped when the chat is closed before the late exit', async () => { + const connection = codex.connections[0]! + const old = closesOnlyOnceExited(connection) + await turnRunning(connection) + forceCloseOnAnUnrecordableFrame(connection) + await vi.waitFor(() => expect(hostSession()?.child?.close?.reported).toBeDefined()) + + // The user closes the chat; that close cannot prove the exit either. + await expect(host.close(SESSION, 'user-close')).rejects.toThrow() + old.exit() + await vi.waitFor(() => expect(hostSession()?.child).toBeNull()) + + await vi.waitFor(async () => expect(await faultRows()).toEqual([FAULT_ROW])) + expect(hostSession()?.lastEndedChild).toMatchObject({ + cause: 'user-close', + failure: { kind: 'hostFault' } + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 398d04873d41..352deffd98d8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -15,6 +15,10 @@ import { type StructuredAgentSessionChildExitContext } from './structured-agent-session-child-exit' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' +import { + recordUnprovenStructuredAgentSessionEnd, + recordUnprovenStructuredAgentSessionEndUnderSerialize +} from './structured-agent-session-unproven-end' export class StructuredAgentSessionEventRecovery { private readonly sinkFailures = new Set() @@ -64,16 +68,32 @@ export class StructuredAgentSessionEventRecovery { const child = this.context.sessions.get(sessionId)?.child const stop = this.context.deps.adapter.forceCloseSession ?? this.context.deps.adapter.closeSession - if (!child || !stop || !(await stopAgentSessionProviderRoot(() => stop(sessionId)))) { + if (!child || !stop) { return } - // Ended in the same step as the stop, so no report of that close can end it first as a - // quiet rest: Orca stopped the provider because its own journal failed. - await this.endExitedChildUnderSerialize(sessionId, child, { - expected: false, + // Orca stopped the provider because its own journal failed. + const end = { reason: `journal sink failure: ${error instanceof Error ? error.message : String(error)}`, failure: agentSessionFailureFact('hostFault') - }) + } + const stopped = await stopAgentSessionProviderRoot(() => stop(sessionId)).catch( + (stopError: unknown) => { + this.context.deps.logger.warn('stopping a provider after its journal failed', { + scope: 'sink-failure-recovery', + sessionId, + error: stopError + }) + return false + } + ) + if (!stopped) { + // Never silent: the child stays on record, closing, and its exit is settled as this end. + recordUnprovenStructuredAgentSessionEndUnderSerialize(this.context, sessionId, child, end) + return + } + // Ended in the same step as the stop, so no report of that close can end it first as a + // quiet rest. + await this.endExitedChildUnderSerialize(sessionId, child, { expected: false, ...end }) }) .catch((error: unknown) => this.context.deps.logger.warn( @@ -95,6 +115,9 @@ export class StructuredAgentSessionEventRecovery { if (event.type === 'started') { return settleStructuredAgentSessionProviderStarted(this.context, event) } + if (event.type === 'end-unproven') { + return recordUnprovenStructuredAgentSessionEnd(this.context, event) + } await settleStructuredAgentSessionChildExit(this.exitContext, event) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 0c63f1fe4b48..ec502566b6f8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -107,7 +107,9 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( return } const cause = 'recorded' in ending ? ending.recorded : ending.cause - if (!child.close) { + // An end the adapter reported unproven began no one's stop: the first stop asked since decides. + const reported = child.close?.cause === 'host-stop' ? child.close.reported : undefined + if (!child.close || (reported && cause !== 'host-stop')) { // Judged before the kill: a stop that ends nothing writes nothing. Its event is issued before // the kill and never awaited by it; the journal writes rows in order. const recorded = (await stopEndsWork(context, sessionId, session, ending)) @@ -117,7 +119,8 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( cause, reason: ('reason' in ending ? ending.reason : undefined) ?? null, recorded, - requestedAt: session.journal.cursor() + requestedAt: session.journal.cursor(), + ...(reported ? { reported } : {}) } } else if (child.close.cause === cause) { // The same stop asked again, such as a second close of the chat, closes what came since. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index ed82fffbe45c..9fcb1bcc9fcb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -49,6 +49,17 @@ export type StructuredAgentSessionChildClose = { /** Where the journal stood when that stop was asked for: the child's end is ordered there, so a * message accepted while the exit was being proven came after it. A repeated ask moves it. */ requestedAt: AgentJournalCursor + /** Set when no stop began this close: the child ended (a fault, an exit) and its own close could + * not prove it gone. The exit that ends the record is settled as this end. In memory only. */ + readonly reported?: StructuredAgentSessionReportedEnd +} + +/** An end the adapter reported but could not prove, kept until its exit is seen or proven. */ +export type StructuredAgentSessionReportedEnd = { + reason: string + failure?: SubmissionRejectionFact + /** Work was in flight when the end was reported, so its settlement shows why it stopped. */ + interruptedWork: boolean } /** The provider process behind a conversation. Written only in diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts new file mode 100644 index 000000000000..e7fc36d35fad --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts @@ -0,0 +1,118 @@ +// An end the adapter could not prove is judged by the same rule an observed exit is: only work that +// was in flight reads as interrupted. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + captureUnfinishedStructuredAgentSessionWork, + settleStructuredAgentSessionDeadGeneration, + unfinishedStructuredAgentSessionWorkWasInterrupted +} from './structured-agent-session-dead-generation-settlement' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { recordUnprovenStructuredAgentSessionEndUnderSerialize } from './structured-agent-session-unproven-end' + +const SESSION = 'session-unproven-end' +const THREAD = 'thread-1' +const NOW = 1_000 +const FAULT = agentSessionFailureFact('hostFault') + +let root: string +let journal: AgentSessionJournal + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-unproven-end-')) + journal = await openAgentSessionJournal({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: THREAD } + }, + database: openTestJournalHostDatabase(root), + now: () => NOW + }) + // The turn finished; only an approval card the agent asked for is left open. + await journal.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 1 }, + { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }, + { fence: 7, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + await journal.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 2 }, + { kind: 'turn', turnId: 'turn-1', state: 'completed', startedAt: 900, completedAt: 950 }, + { fence: 7, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) +}) + +afterEach(async () => { + await journal.close() + await rm(root, { recursive: true, force: true }) +}) + +function statusRows() { + return journal.snapshot().items.filter((item) => item.body.kind === 'status') +} + +it('shows no fault row for an idle open approval when the exit was observed', async () => { + const before = captureUnfinishedStructuredAgentSessionWork(journal) + const show = unfinishedStructuredAgentSessionWorkWasInterrupted(before, journal, NOW) + + await settleStructuredAgentSessionDeadGeneration({ + journal, + sessionId: SESSION, + fence: 7, + settlementId: `provider-exit:${SESSION}:7:generation-1`, + pendingSubmissionReason: 'provider_exited_before_acknowledgement', + verdict: { state: 'interrupted', completedAt: NOW }, + showUnexpectedExitOutcome: show, + exitFailure: FAULT + }) + + expect(show).toBe(false) + expect(statusRows()).toEqual([]) +}) + +function reportOn(journalOf: AgentSessionJournal) { + const session: Pick = { + child: { generation: 'generation-1', fence: 7, phase: 'ready' }, + journal: journalOf + } + recordUnprovenStructuredAgentSessionEndUnderSerialize( + { sessions: new Map([[SESSION, session]]), now: () => NOW }, + SESSION, + { generation: 'generation-1', fence: 7 }, + { reason: 'journal sink failure', failure: FAULT } + ) + return session.child?.close +} + +it('judges the same idle approval not interrupted when the end could not be proven', () => { + expect(reportOn(journal)).toMatchObject({ + cause: 'host-stop', + reported: { reason: 'journal sink failure', failure: FAULT, interruptedWork: false } + }) +}) + +it('judges a turn still running when the end was reported interrupted', async () => { + await journal.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'turn-2', ordinal: 1 }, + { kind: 'turn', turnId: 'turn-2', state: 'running', startedAt: 990 }, + { fence: 7, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + + expect(reportOn(journal)?.reported).toMatchObject({ interruptedWork: true }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts new file mode 100644 index 000000000000..0fe8c8b95b23 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts @@ -0,0 +1,73 @@ +// An end the adapter could not prove. The child may still be running, so the host keeps it on record +// and closing: nothing writes to it or starts beside it, and every later send, start or Stop joins +// that close and retries the kill. The exit that finally ends the record is settled as the end that +// was reported, not as a stop anyone asked for. + +import type { StructuredAgentSessionEndUnprovenEvent } from './structured-agent-session-adapter' +import { + captureUnfinishedStructuredAgentSessionWork, + unfinishedStructuredAgentSessionWorkWasInterrupted +} from './structured-agent-session-dead-generation-settlement' +import type { + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity, + StructuredAgentSessionReportedEnd +} from './structured-agent-session-host-types' +import { sameProviderChild } from './structured-agent-session-provider-child' + +type UnprovenEndSession = Pick + +export type StructuredAgentSessionUnprovenEndContext = { + sessions: Map + now: () => number + publishStatus?: (sessionId: string) => void + serialize: (sessionId: string, task: () => Promise) => Promise +} + +/** For a caller inside the session's serialize. A report for any other child is stale. */ +export function recordUnprovenStructuredAgentSessionEndUnderSerialize( + context: Omit, + sessionId: string, + ended: StructuredAgentSessionProviderChildIdentity, + report: Pick +): void { + const session = context.sessions.get(sessionId) + const child = session?.child + if (!session || !child || !sameProviderChild(child, ended)) { + return + } + // A close a stop already began keeps its cause: it is that stop's to end. + if (!child.close) { + child.close = { + cause: 'host-stop', + reason: report.reason, + recorded: Promise.resolve(), + requestedAt: session.journal.cursor(), + reported: { + reason: report.reason, + ...(report.failure ? { failure: report.failure } : {}), + // The rule an observed exit uses: an approval or question left open is not interrupted. + interruptedWork: unfinishedStructuredAgentSessionWorkWasInterrupted( + captureUnfinishedStructuredAgentSessionWork(session.journal), + session.journal, + context.now() + ) + } + } + } + context.publishStatus?.(sessionId) +} + +export function recordUnprovenStructuredAgentSessionEnd( + context: StructuredAgentSessionUnprovenEndContext, + event: StructuredAgentSessionEndUnprovenEvent +): Promise { + return context.serialize(event.sessionId, async () => + recordUnprovenStructuredAgentSessionEndUnderSerialize( + context, + event.sessionId, + { generation: event.acquisitionGeneration, fence: event.fence }, + { reason: event.reason, ...(event.failure ? { failure: event.failure } : {}) } + ) + ) +} diff --git a/src/main/runtime/claude-structured-startup-fault-is-not-an-exit.test.ts b/src/main/runtime/claude-structured-startup-fault-is-not-an-exit.test.ts index 863cbaeaebf9..f79ab0ed75dc 100644 --- a/src/main/runtime/claude-structured-startup-fault-is-not-an-exit.test.ts +++ b/src/main/runtime/claude-structured-startup-fault-is-not-an-exit.test.ts @@ -136,4 +136,38 @@ describe('a Claude start that Orca fails while the CLI is still running', () => ) expect(await failureRows(host)).toEqual([{ text: STOPPED_TEXT, kind: 'providerStartFailed' }]) }) + + it('rejects every message held on a start Orca failed with that start, even when its close could not prove the CLI gone', async () => { + claude.behave(SESSION, { initHangs: true, initNamesForeignSession: true, closeUnproven: true }) + const host = await claude.install() + await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ + ok: true + }) + const first = await send(host, 'hello') + const second = await send(host, 'and this') + + claude.child(SESSION).answerInit() + await waitForStructuredAgentSessionRecovery() + // The CLI may still run, so the chat waits on it; once it exits, the start that failed decides. + await vi.waitFor(() => + expect(host['sessions'].get(SESSION)?.child?.close?.reported).toBeDefined() + ) + claude.child(SESSION).exit(scriptedClaudeExitError(DIAGNOSTIC)) + await released(host) + + for (const held of [first, second]) { + await vi.waitFor(async () => + expect(await submission(host, held)).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringMatching(/^Claude couldn't start\./), + rejection: { kind: 'startFailed' } + }) + ) + } + expect(claude.child(SESSION).calls).not.toContain('send') + expect(host['sessions'].get(SESSION)?.lastEndedChild).toMatchObject({ + cause: 'exit', + failure: { kind: 'startFailed' } + }) + }) }) diff --git a/src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts b/src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts index b236c81d594b..fd05b2cd9628 100644 --- a/src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts +++ b/src/main/runtime/structured-agent-session-lifecycle-delivery.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import { recordingStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { createStructuredAgentSessionLifecycleDelivery } from './structured-agent-session-lifecycle-delivery' @@ -46,4 +46,40 @@ describe('provider exits reaching the host', () => { await delivery.drain() expect(handled).toEqual(['closed', 'crashed']) }) + + it("keeps an unproven end off the exit chain, so its wait on its own session holds up no other session's exit, and drain still waits for it", async () => { + const sessionLane = Promise.withResolvers() + const handled: string[] = [] + const delivery = createStructuredAgentSessionLifecycleDelivery({ + handle: async (event) => { + if (event.type === 'end-unproven') { + await sessionLane.promise + } + handled.push(`${event.type}:${event.sessionId}`) + }, + logger: recordingStructuredAgentSessionLogger().logger, + drainObservedExits: async () => {} + }) + + delivery.deliver({ + type: 'end-unproven', + sessionId: 'session-a', + reason: 'journal fault', + fence: 1, + acquisitionGeneration: 'a-1' + }) + delivery.deliver(ended('session-b', 'unexpected-exit')) + await vi.waitFor(() => expect(handled).toEqual(['ended:session-b'])) + + let drained = false + const draining = delivery.drain().then(() => { + drained = true + }) + await new Promise((resolve) => setImmediate(resolve)) + expect(drained).toBe(false) + + sessionLane.resolve() + await draining + expect(handled).toEqual(['ended:session-b', 'end-unproven:session-a']) + }) }) diff --git a/src/main/runtime/structured-agent-session-lifecycle-delivery.ts b/src/main/runtime/structured-agent-session-lifecycle-delivery.ts index ba307c2cfb0a..3144afa041f5 100644 --- a/src/main/runtime/structured-agent-session-lifecycle-delivery.ts +++ b/src/main/runtime/structured-agent-session-lifecycle-delivery.ts @@ -3,13 +3,19 @@ // Exit recovery runs on one chain so teardown can drain it: exit callbacks arrive from child // process tasks, and a fire-and-forget one could otherwise append after the host flushed and // removed its journal directory. That chain orders nothing across sessions, and a recovery on it -// can run a whole reacquisition, so `started` and the end of a close the host asked for stay off -// it: each takes only its own session's serialized step, and is tracked here so the same drain -// still waits for it. +// can run a whole reacquisition, so `started`, an end whose close is unproven and the end of a +// close the host asked for stay off it: each takes only its own session's serialized step, and is +// tracked here so the same drain still waits for it. import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' +const LOG_SCOPES = { + started: 'lifecycle-started', + ended: 'lifecycle-exit', + 'end-unproven': 'lifecycle-end-unproven' +} as const satisfies Record + export function createStructuredAgentSessionLifecycleDelivery(input: { handle: (event: StructuredAgentSessionLifecycleEvent) => Promise | undefined logger: StructuredAgentSessionLogger @@ -27,7 +33,7 @@ export function createStructuredAgentSessionLifecycleDelivery(input: { await input.handle(event) } catch (error) { input.logger.warn(`delivering a provider ${event.type} event to the host failed`, { - scope: event.type === 'started' ? 'lifecycle-started' : 'lifecycle-exit', + scope: LOG_SCOPES[event.type], sessionId: event.sessionId, error }) @@ -35,7 +41,7 @@ export function createStructuredAgentSessionLifecycleDelivery(input: { } return { deliver: (event) => { - if (event.type === 'started' || event.cause === 'requested-close') { + if (event.type !== 'ended' || event.cause === 'requested-close') { // Called now, so the step is queued on its own session's lane, behind nothing of another's. const settling = settle(event) settlingStarts.add(settling) diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index b5efd11efad1..23f220471938 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -52,6 +52,7 @@ import { createStructuredAgentEnvironmentResolvers } from './structured-agent-sh import type { ClaudeStructuredAuthPolicy } from '../claude-accounts/claude-structured-auth-policy' import { createStructuredClaudeRuntimeAdapter } from './structured-claude-runtime-adapter' import { createStructuredAgentSessionLifecycleDelivery } from './structured-agent-session-lifecycle-delivery' +import { structuredCodexLifecycleEvent } from './structured-codex-lifecycle-event' import { createStructuredAgentSessionDispatchFollowUps } from './structured-agent-session-dispatch-followups' import { agentModelCatalogStore } from '../native-chat/agent-model-catalog/agent-model-catalog-store' import { @@ -272,9 +273,9 @@ async function installOnJournal( onPrimaryThreadStoppedRunning: releaseUnansweredDispatches, logger: deps.logger, onEvent: (event) => { - // Every exit, expected or not: the host ends that child's record. - if (event.type === 'ended' && 'cause' in event) { - lifecycle.deliver(event) + const lifecycleEvent = structuredCodexLifecycleEvent(event) + if (lifecycleEvent) { + lifecycle.deliver(lifecycleEvent) } } }) diff --git a/src/main/runtime/structured-claude-runtime-adapter.ts b/src/main/runtime/structured-claude-runtime-adapter.ts index 732515cdde60..af9fcec53298 100644 --- a/src/main/runtime/structured-claude-runtime-adapter.ts +++ b/src/main/runtime/structured-claude-runtime-adapter.ts @@ -45,7 +45,7 @@ export type StructuredClaudeRuntimeAdapterDeps = { export function structuredClaudeLifecycleEvent( event: ClaudeStructuredSessionEvent ): StructuredAgentSessionLifecycleEvent | null { - if (event.type === 'started') { + if (event.type === 'started' || event.type === 'end-unproven') { return event } // Every exit of a child with an identity, expected or not: the host ends that child's record. diff --git a/src/main/runtime/structured-codex-lifecycle-event.ts b/src/main/runtime/structured-codex-lifecycle-event.ts new file mode 100644 index 000000000000..4773ce65bbb6 --- /dev/null +++ b/src/main/runtime/structured-codex-lifecycle-event.ts @@ -0,0 +1,12 @@ +import type { CodexStructuredSessionEvent } from '../codex/codex-structured-session-state' +import type { StructuredAgentSessionLifecycleEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' + +/** The Codex adapter events the host's lifecycle handler consumes: every exit, expected or not, and + * an end whose close could not prove the exit. */ +export function structuredCodexLifecycleEvent( + event: CodexStructuredSessionEvent +): StructuredAgentSessionLifecycleEvent | null { + return event.type === 'end-unproven' || (event.type === 'ended' && 'cause' in event) + ? event + : null +} From b9c4247a6d93a36293f22abeaf91ad3b4e4d3bb1 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:27:37 -0700 Subject: [PATCH 18/22] fix(native-chat): a stop that takes over a fault's close writes no Stop event; review round 3 - A Stop, chat close or quit that takes over the close an unproven fault began keeps its own cause but writes no Stop event, like any stop that joins a close: the fault ended the turn, so it reads interrupted with the fault's row, never the user's cancellation. - A /compact rejected because its agent was already gone or never started keeps that reason on its row instead of reading only "Compaction failed." --- ...session-claude-unproven-fault-send.test.ts | 49 ++++++++++++------- ...ession-codex-forced-close-unproven.test.ts | 48 ++++++++++++------ .../structured-agent-session-command-turn.ts | 11 ++++- .../structured-agent-session-host-lifetime.ts | 13 +++-- ...structured-conversation-compaction.test.ts | 25 ++++++++++ 5 files changed, 106 insertions(+), 40 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts index c7621021bb4e..5056b003e547 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts @@ -412,6 +412,7 @@ it.each([ await eventually(() => expect(hostSession()?.child).toBeNull()) // The same end reads the same however long its proof took. + expect(await turnRows()).toEqual([{ state: 'interrupted', outcome: undefined }]) expect(await faultRows()).toEqual([FAULT_ROW]) expect(hostSession()?.lastEndedChild).toMatchObject({ cause: 'exit', @@ -419,22 +420,36 @@ it.each([ }) }) -it("still tells why a turn a fault cut short stopped when the user's Stop comes before the late exit", async () => { - const connection = claude.connections[0]! - await turnRunning(connection) - await faultedUnproven(connection) - expect(hostSession()?.child?.close?.reported).toBeDefined() +async function turnRows() { + await host.flushStreamedEvents(SESSION) + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'turn' ? [{ state: item.body.state, outcome: item.body.outcome }] : [] + ) +} - // The turn still reads running, so the user presses Stop; its close cannot prove the exit either. - await expect(stop()).resolves.toMatchObject({ ok: true }) - await laneDrained() - exits(connection) - await eventually(() => expect(hostSession()?.child).toBeNull()) +it.each([ + ['a Stop', 'user-stop'], + ['a chat close', 'user-close'] +] as const)( + 'still tells why a turn a fault cut short stopped when %s comes before the late exit, as the fault alone would', + async (_name, cause) => { + const connection = claude.connections[0]! + await turnRunning(connection) + await faultedUnproven(connection) + expect(hostSession()?.child?.close?.reported).toBeDefined() - expect(await faultRows()).toEqual([FAULT_ROW]) - // The Stop stays the user's; the fault that came first says why the turn ended. - expect(hostSession()?.lastEndedChild).toMatchObject({ - cause: 'user-stop', - failure: { kind: 'hostFault' } - }) -}) + // The turn still reads running, so the user acts; that close cannot prove the exit either. + await (cause === 'user-stop' + ? expect(stop()).resolves.toMatchObject({ ok: true }) + : expect(host.close(SESSION, 'user-close')).rejects.toThrow()) + await laneDrained() + exits(connection) + await eventually(() => expect(hostSession()?.child).toBeNull()) + + // The fault ended the turn, so it reads interrupted, never the user's cancellation, with the + // fault's one row; the end stays the user's. + expect(await turnRows()).toEqual([{ state: 'interrupted', outcome: undefined }]) + expect(await faultRows()).toEqual([FAULT_ROW]) + expect(hostSession()?.lastEndedChild).toMatchObject({ cause, failure: { kind: 'hostFault' } }) + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts index a402192d7a1d..0f353ca5bf78 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts @@ -252,27 +252,43 @@ it.each([ // The same end reads the same however long its proof took. await vi.waitFor(async () => expect(await faultRows()).toEqual([FAULT_ROW])) + expect(await turnRows()).toEqual([{ state: 'interrupted', outcome: undefined }]) expect(hostSession()?.lastEndedChild).toMatchObject({ cause: 'exit', failure: { kind: 'hostFault' } }) }) -it('still tells why a turn a forced close cut short stopped when the chat is closed before the late exit', async () => { - const connection = codex.connections[0]! - const old = closesOnlyOnceExited(connection) - await turnRunning(connection) - forceCloseOnAnUnrecordableFrame(connection) - await vi.waitFor(() => expect(hostSession()?.child?.close?.reported).toBeDefined()) +async function turnRows() { + await host.flushStreamedEvents(SESSION) + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'turn' ? [{ state: item.body.state, outcome: item.body.outcome }] : [] + ) +} - // The user closes the chat; that close cannot prove the exit either. - await expect(host.close(SESSION, 'user-close')).rejects.toThrow() - old.exit() - await vi.waitFor(() => expect(hostSession()?.child).toBeNull()) +it.each([ + ['the chat is closed', 'user-close'], + ['Orca evicts it', 'evict'] +] as const)( + 'still tells why a turn a forced close cut short stopped when %s before the late exit, as the fault alone would', + async (_name, cause) => { + const connection = codex.connections[0]! + const old = closesOnlyOnceExited(connection) + await turnRunning(connection) + forceCloseOnAnUnrecordableFrame(connection) + await vi.waitFor(() => expect(hostSession()?.child?.close?.reported).toBeDefined()) + const journal = hostSession()!.journal + const stopEvents = vi.spyOn(journal, 'appendStopEvent') - await vi.waitFor(async () => expect(await faultRows()).toEqual([FAULT_ROW])) - expect(hostSession()?.lastEndedChild).toMatchObject({ - cause: 'user-close', - failure: { kind: 'hostFault' } - }) -}) + // That close cannot prove the exit either. + await expect(host.close(SESSION, cause)).rejects.toThrow() + old.exit() + await vi.waitFor(() => expect(hostSession()?.child).toBeNull()) + + await vi.waitFor(async () => expect(await faultRows()).toEqual([FAULT_ROW])) + // The fault ended the turn: it reads interrupted, never the user's cancellation. + expect(await turnRows()).toEqual([{ state: 'interrupted', outcome: undefined }]) + expect(stopEvents).not.toHaveBeenCalled() + expect(hostSession()?.lastEndedChild).toMatchObject({ cause, failure: { kind: 'hostFault' } }) + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts index b1654ff70790..c9d3ff8705f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts @@ -28,6 +28,7 @@ import { type AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionConversationCommand } from '../../../shared/agent-session-conversation-command' +import { classifyDispatchRejection } from '../../../shared/structured-agent-session-dispatch-rejection' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import { agentSessionRefusalReference } from '../../../shared/agent-session-wire-refusals' @@ -224,12 +225,18 @@ export async function handOverStructuredAgentSessionCommand( return } if (admission.state === 'rejected') { - // The provider refused the compaction itself: its row reads as the compaction failing. + // A child that was already gone, or never started, keeps its own reason (the adapter had let + // it go); any other refusal reads as the compaction failing. + const { category, kind } = classifyDispatchRejection(admission) + const agentGone = + category === 'undelivered' || category === 'startFailed' || kind === 'hostFault' await settleUnsentCommand( ctx, clientMessageId, admission, - agentSessionFailureFact('compactionFailed', { detail: admission.rejection.detail }) + agentGone + ? undefined + : agentSessionFailureFact('compactionFailed', { detail: admission.rejection.detail }) ) } else if (admission.state !== 'admitted') { // An unknown write leaves the turn to the provider's end or the child's: it may have run. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index ec502566b6f8..7e9e56267a7b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -94,7 +94,8 @@ export async function abandonQueuedStructuredAgentSessionMessages( * already begun, and waits for the exit's proof as long as a caller may. Still unproven, it throws * and the close keeps running: a later proof, or the next asker's attempt, ends the record. * `ending` is how the child's end is told: a user's Stop, the host stopping it for a cause (with - * its text), or an eviction the conversation's close follows. The first stop's ending decides. + * its text), or an eviction the conversation's close follows. The first stop's ending decides; a + * fault's unproven end is no one's stop, so the first stop after it takes its cause, writing no event. */ export async function stopStructuredAgentSessionAgentUnderSerialize( context: StructuredAgentSessionLifetimeContext, @@ -111,10 +112,12 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( const reported = child.close?.cause === 'host-stop' ? child.close.reported : undefined if (!child.close || (reported && cause !== 'host-stop')) { // Judged before the kill: a stop that ends nothing writes nothing. Its event is issued before - // the kill and never awaited by it; the journal writes rows in order. - const recorded = (await stopEndsWork(context, sessionId, session, ending)) - ? recordStopEvent(context, sessionId, session, ending) - : Promise.resolve() + // the kill and never awaited by it; the journal writes rows in order. The fault already ended + // the work a reported end's taker would name, so like any stop that joins a close it writes none. + const recorded = + !reported && (await stopEndsWork(context, sessionId, session, ending)) + ? recordStopEvent(context, sessionId, session, ending) + : Promise.resolve() child.close = { cause, reason: ('reason' in ending ? ending.reason : undefined) ?? null, diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts b/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts index 22766f28a416..56bc62a7defa 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts @@ -332,6 +332,31 @@ it('says only that the compaction failed when the provider refused it without wo ) }) +it('keeps the reason when the agent the compaction was meant for had already gone', async () => { + await attach() + // As the adapter answers for a child it let go after a fault Orca could not prove ended. + compact.mockResolvedValue({ + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('hostFault'), { surface: 'rejection' }) + }) + await state.host.conversationCommand(CALLER, compactParams()) + + await vi.waitFor(async () => + expect( + (await journal()).items + .filter((item) => item.body.kind === 'status' && item.body.tone === 'error') + .map((item) => item.body) + ).toEqual([ + { + kind: 'status', + text: "Orca ran into a problem, so this didn't go through. Try again.", + failure: { kind: 'hostFault' }, + tone: 'error' + } + ]) + ) +}) + it('refuses the command at handover when the provider opened a turn meanwhile (B3)', async () => { await attach() const events = state.acquire.mock.calls.at(-1)?.[0].events From 96416f316aa4dff0a997a9c997e001e0cb05ef73 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:32:55 -0700 Subject: [PATCH 19/22] test(native-chat): pin where a chat close that takes over a fault's close ends the child A message accepted while a fault's close is unproven, then a chat close that takes that close over and whose own rejection of the queue fails, then the exit's proof: the end stands where the close was asked, so the message is closed with the chat and no new agent starts (Claude and Codex). --- ...session-claude-unproven-fault-send.test.ts | 42 ++++++++++++++++++ ...ession-codex-forced-close-unproven.test.ts | 44 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts index 5056b003e547..a2d99c84292a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts @@ -453,3 +453,45 @@ it.each([ expect(hostSession()?.lastEndedChild).toMatchObject({ cause, failure: { kind: 'hostFault' } }) } ) + +it("closes a message accepted while a fault's close was unproven when the chat close that takes it over is proven later", async () => { + const connection = claude.connections[0]! + await turnWithBackgroundTask(connection) + await faultedUnproven(connection) + const closing = hostSession()!.child! + const faultAt = closing.close!.requestedAt + + // A message is accepted, then the chat is closed, then the exit is proven, all before the + // message's delivery step runs. + const held = Promise.withResolvers() + const holding = host['tasks'].serialize(SESSION, () => held.promise) + const sent = send('Sent before the close.') + // The close's own rejection of what is queued fails, so only where its end stands closes it. + vi.spyOn(hostSession()!.journal, 'rejectQueuedSubmissions').mockRejectedValueOnce( + new Error('disk full') + ) + const closed = host.close(SESSION, 'user-close') + const ended = host['tasks'].serialize(SESSION, () => + host['eventRecovery'].endExitedChildUnderSerialize(SESSION, closing, { + expected: true, + reason: 'claude session closed' + }) + ) + held.resolve() + await holding + const message = await sent + await expect(closed).rejects.toThrow() + await ended + + // The close took the fault's close over where it was asked, after the message: the end stands + // there, so the message is closed with the chat and starts no new agent. + expect(await settled(message)).toMatchObject({ + dispatchState: 'rejected', + rejection: { kind: 'chatClosed' } + }) + const end = hostSession()!.lastEndedChild! + expect(end).toMatchObject({ cause: 'user-close', failure: { kind: 'hostFault' } }) + expect(end.endedAt.sequence).toBeGreaterThan(faultAt.sequence) + expect(claude.connections).toHaveLength(1) + expect(wrote(connection, 'Sent before the close.')).toBe(false) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts index 0f353ca5bf78..cc4f0c950e9d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts @@ -292,3 +292,47 @@ it.each([ expect(hostSession()?.lastEndedChild).toMatchObject({ cause, failure: { kind: 'hostFault' } }) } ) + +it('closes a message accepted while a forced close was unproven when the chat close that takes it over is proven later', async () => { + const connection = codex.connections[0]! + closesOnlyOnceExited(connection) + forceCloseOnAnUnrecordableFrame(connection) + await vi.waitFor(() => expect(hostSession()?.child?.close?.reported).toBeDefined()) + const closing = hostSession()!.child! + const faultAt = closing.close!.requestedAt + + // A message is accepted, then the chat is closed, then the exit is proven, all before the + // message's delivery step runs. + const held = Promise.withResolvers() + const holding = host['tasks'].serialize(SESSION, () => held.promise) + const sent = send('Sent before the close.') + // The close's own rejection of what is queued fails, so only where its end stands closes it. + vi.spyOn(hostSession()!.journal, 'rejectQueuedSubmissions').mockRejectedValueOnce( + new Error('disk full') + ) + const closed = host.close(SESSION, 'user-close') + const ended = host['tasks'].serialize(SESSION, () => + host['eventRecovery'].endExitedChildUnderSerialize(SESSION, closing, { + expected: true, + reason: 'codex session closed' + }) + ) + held.resolve() + await holding + const message = await sent + await expect(closed).rejects.toThrow() + await ended + + // The end stands where the close took over, after the message, so it is closed with the chat. + await vi.waitFor(async () => + expect(await submission(message)).toMatchObject({ + dispatchState: 'rejected', + rejection: { kind: 'chatClosed' } + }) + ) + const end = hostSession()!.lastEndedChild! + expect(end).toMatchObject({ cause: 'user-close', failure: { kind: 'hostFault' } }) + expect(end.endedAt.sequence).toBeGreaterThan(faultAt.sequence) + expect(codex.connections).toHaveLength(1) + expect(startedTurnWith(connection, 'Sent before the close.')).toBe(false) +}) From aca48b4a2b84b168871768b2344a6a30a5aeb63f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:35:32 -0700 Subject: [PATCH 20/22] docs(native-chat): say a stop that takes over a fault's close keeps its own cause --- .../structured-agent-session-host-lifetime.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 7e9e56267a7b..6c2102ba9b1a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -95,7 +95,8 @@ export async function abandonQueuedStructuredAgentSessionMessages( * and the close keeps running: a later proof, or the next asker's attempt, ends the record. * `ending` is how the child's end is told: a user's Stop, the host stopping it for a cause (with * its text), or an eviction the conversation's close follows. The first stop's ending decides; a - * fault's unproven end is no one's stop, so the first stop after it takes its cause, writing no event. + * fault's unproven end is no one's stop, so the first stop after it takes over that close with its + * own cause, keeping the fault's details and writing no event. */ export async function stopStructuredAgentSessionAgentUnderSerialize( context: StructuredAgentSessionLifetimeContext, From 4081e1f9bfdcfd373a54c289d0ea5d3d8dd4ea81 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:49:21 -0700 Subject: [PATCH 21/22] fix(native-chat): an unproven end's close records no Stop event as null, as main's close type now says --- .../agent-session-wire/structured-agent-session-unproven-end.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts index 0fe8c8b95b23..560c2657d10d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.ts @@ -41,7 +41,7 @@ export function recordUnprovenStructuredAgentSessionEndUnderSerialize( child.close = { cause: 'host-stop', reason: report.reason, - recorded: Promise.resolve(), + recorded: Promise.resolve(null), requestedAt: session.journal.cursor(), reported: { reason: report.reason, From f5e48b6f2c539e5a0f5b6d4b0e9fee782c21ebd8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 02:30:08 -0700 Subject: [PATCH 22/22] test(native-chat): build the unproven-end tests' host and handles as main now does main's host takes its agents from a registry and the provider handle is now transport/agent/nativeId; the three new test files follow both. --- ...tructured-agent-session-claude-unproven-fault-send.test.ts | 2 ++ ...ructured-agent-session-codex-forced-close-unproven.test.ts | 4 +++- .../structured-agent-session-unproven-end.test.ts | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts index a2d99c84292a..2438c2ba5093 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-unproven-fault-send.test.ts @@ -21,6 +21,7 @@ import type { AgentSessionRecordStore } from '../../runtime/agent-session-record import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { claudeAndCodexAgents } from './structured-agent-session-adapter-router-test-support' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' import { @@ -79,6 +80,7 @@ beforeEach(async () => { }) store = await openTestAgentSessionRecordStore(root) host = new StructuredAgentSessionHost({ + agents: claudeAndCodexAgents(adapter), store, adapter: Object.assign(adapter, { supportsCreate: () => true }), journalDatabase: openTestJournalHostDatabase(root), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts index cc4f0c950e9d..df652508f0f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-codex-forced-close-unproven.test.ts @@ -18,6 +18,7 @@ import type { AgentSessionRecordStore } from '../../runtime/agent-session-record import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' import { structuredCodexLifecycleEvent } from '../../runtime/structured-codex-lifecycle-event' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { claudeAndCodexAgents } from './structured-agent-session-adapter-router-test-support' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' import { @@ -56,7 +57,7 @@ beforeEach(async () => { args: ['app-server'], cwd: root, codexHome: null, - resumeThreadId: head?.handle.provider === 'codex' ? head.handle.threadId : null + resumeThreadId: head?.handle.nativeId ?? null } }, // As the runtime routes them. @@ -74,6 +75,7 @@ beforeEach(async () => { }) store = await openTestAgentSessionRecordStore(root) host = new StructuredAgentSessionHost({ + agents: claudeAndCodexAgents(adapter), store, adapter: Object.assign(adapter, { supportsCreate: () => true }), journalDatabase: openTestJournalHostDatabase(root), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts index e7fc36d35fad..073f0e4809c9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unproven-end.test.ts @@ -34,7 +34,7 @@ beforeEach(async () => { workspaceId: 'workspace-1', hostId: 'local', agent: 'codex', - providerHandle: { kind: 'codex', threadId: THREAD } + providerHandle: { transport: 'codex-app-server', agent: 'codex', nativeId: THREAD } }, database: openTestJournalHostDatabase(root), now: () => NOW