diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1b52aac7571d..6125304af8bf 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -757,6 +757,7 @@ jobs: tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts tests/e2e/cross-version-wire/cross-version-worktree-identity-downgrade.unit.test.ts tests/e2e/cross-version-wire/cross-version-session-tabs-retirement-proof.unit.test.ts + tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts managed_hook_node18: name: managed hooks on Node 18 diff --git a/config/scripts/pr-code-change-scope.mjs b/config/scripts/pr-code-change-scope.mjs index babe8e8c7e05..b1a8e17726a1 100644 --- a/config/scripts/pr-code-change-scope.mjs +++ b/config/scripts/pr-code-change-scope.mjs @@ -147,6 +147,8 @@ const CROSS_VERSION_WIRE_PREFIXES = [ 'src/main/native-chat/agent-session-journal/', 'src/main/native-chat/agent-session-wire/', 'src/main/runtime/agent-session-record-store', + 'src/main/runtime/agent-session-recovery-capsule', + 'src/shared/agent-session-resume-marker', 'src/main/runtime/rpc/dispatcher', 'src/main/runtime/rpc/methods/agent-launch', 'src/main/runtime/rpc/methods/ai-vault.ts', diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 4dff79d6666a..8d91532da288 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -119,7 +119,11 @@ forgot the row would strand a permanently working-looking agent. Two rules the ingest must keep: - **Never persist a structured row.** The journal is the durable truth for a - structured session and the host republishes on restore. A structured row in + structured session and the host republishes on restore: for each listed chat, + from its saved settled status when its journal still stands at the position + that status was computed at, and otherwise from the journal. That saved copy + is the host's own cache (`agent-session-saved-status.db`), never this store's + file. A structured row in `last-status.json` would hydrate as `restoredUnconfirmed` and then fight the live republish. The serializer skips rows carrying `structuredHost`, and hydrate drops any such row found on disk. Applying one therefore also skips diff --git a/mobile/src/session/use-mobile-structured-agent-session-hold-refused.test.tsx b/mobile/src/session/use-mobile-structured-agent-session-hold-refused.test.tsx new file mode 100644 index 000000000000..cba5ec91ba84 --- /dev/null +++ b/mobile/src/session/use-mobile-structured-agent-session-hold-refused.test.tsx @@ -0,0 +1,103 @@ +import { createElement } from 'react' +import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { AgentSessionSubscribeEvent } from '../../../src/shared/agent-session-wire' +import type { RpcClient } from '../transport/rpc-client' +import { useMobileStructuredAgentSession } from './use-mobile-structured-agent-session' + +const asyncStorage = vi.hoisted(() => ({ + getItem: vi.fn(async () => null), + setItem: vi.fn(async () => undefined), + removeItem: vi.fn(async () => undefined) +})) + +vi.mock('@react-native-async-storage/async-storage', () => ({ default: asyncStorage })) + +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a literal snapshot frame; the hook reads its type, page and fence. +const SNAPSHOT: AgentSessionSubscribeEvent = { + type: 'snapshot', + sessionId: 'session-1', + fence: 3, + page: { + sessionId: 'session-1', + epoch: 'epoch-1', + direction: 'tail', + items: [ + { + itemId: 'msg-1', + revision: 1, + sequence: 1, + observedAt: 10, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'still here' }] } + } + ], + removedItemIds: [], + submissions: [], + window: { + oldest: { epoch: 'epoch-1', sequence: 1 }, + newest: { epoch: 'epoch-1', sequence: 1 }, + nextCursor: { epoch: 'epoch-1', sequence: 2 } + }, + liveCursor: { epoch: 'epoch-1', sequence: 1 }, + hasOlder: false, + hasNewer: false + } +} as AgentSessionSubscribeEvent + +let renderer: ReactTestRenderer | null = null +let hook: ReturnType | null = null + +afterEach(() => { + act(() => renderer?.unmount()) + renderer = null + hook = null +}) + +beforeEach(() => { + vi.clearAllMocks() +}) + +it('subscribes when an older host refuses the hold, and shows the transcript (P2-24)', async () => { + const sendRequest = vi.fn(async (method: string) => + method === 'agentSession.hold' + ? { + ok: false, + error: { code: 'agent_session_owner_restart_failed', message: 'Codex could not start.' }, + _meta: { runtimeId: 'runtime-1' } + } + : { ok: true, result: {}, _meta: { runtimeId: 'runtime-1' } } + ) + const subscribe = vi.fn((_method: string, _params: unknown, onData: (value: unknown) => void) => { + onData(SNAPSHOT) + return () => undefined + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: an RPC client stub with the two members the hook calls. + const client = { sendRequest, subscribe } as unknown as RpcClient + function Harness(): null { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the hook's options, of which this test supplies the ones it reads. + hook = useMobileStructuredAgentSession({ + client, + sessionId: 'session-1', + sourceIdentity: 'host-a\0workspace-a', + enabled: true, + connected: true, + agent: 'codex', + onSendError: () => undefined + } as never) + return null + } + + act(() => { + renderer = create(createElement(Harness)) + }) + + await vi.waitFor(() => + expect(subscribe).toHaveBeenCalledWith( + 'agentSession.subscribe', + { sessionId: 'session-1' }, + expect.any(Function) + ) + ) + await vi.waitFor(() => expect(hook?.session.messages.length).toBeGreaterThan(0)) + expect(hook?.session.status).not.toBe('error') +}) diff --git a/mobile/src/session/use-mobile-structured-agent-state.ts b/mobile/src/session/use-mobile-structured-agent-state.ts index 494260049689..d2b6d1a50629 100644 --- a/mobile/src/session/use-mobile-structured-agent-state.ts +++ b/mobile/src/session/use-mobile-structured-agent-state.ts @@ -19,6 +19,33 @@ const MAX_RETAINED_SESSION_STATES = 32 /** Bounded so a busy stream cannot turn one Load-earlier tap into an endless read chain. */ const OLDER_PAGE_ANCHOR_ATTEMPTS = 3 +/** + * Opens the transcript stream once the hold settles, either way: a refused hold is an older host + * saying it could not start the agent — which the next send does — never a reason to hide the + * transcript. Returns what ends the stream, opened or not yet. Outside the effect so its cleanup + * rule can see the stream is owned. + */ +function openTranscriptAfterHold( + client: RpcClient, + sessionId: string, + held: Promise, + onFrame: (raw: unknown) => void +): () => void { + let ended = false + let close = (): void => {} + void held + .catch(() => undefined) + .then(() => { + if (!ended) { + close = client.subscribe('agentSession.subscribe', { sessionId }, onFrame) + } + }) + return () => { + ended = true + close() + } +} + function isSubscribeEvent(value: unknown): value is AgentSessionSubscribeEvent { if (typeof value !== 'object' || value === null) { return false @@ -101,39 +128,21 @@ export function useMobileStructuredAgentState(args: { } apply({ type: 'loading' }) const holderId = structuredAgentSessionHolderId('mobile-chat') - let cancelled = false - let unsubscribe = (): void => {} const held = callAgentSession(client, 'agentSession.hold', { sessionId, holderId }) - void held - .then(() => { - if (cancelled) { - return - } - unsubscribe = client.subscribe('agentSession.subscribe', { sessionId }, (raw) => { - if ( - typeof raw === 'object' && - raw !== null && - (raw as { type?: unknown }).type === 'error' - ) { - apply({ type: 'error', message: String((raw as { message?: unknown }).message ?? '') }) - return - } - if (isSubscribeEvent(raw)) { - apply({ type: 'event', event: raw }) - } - }) - }) - .catch((error: unknown) => { - if (!cancelled) { - apply({ type: 'error', message: error instanceof Error ? error.message : String(error) }) - } - }) + const endStream = openTranscriptAfterHold(client, sessionId, held, (raw) => { + if (typeof raw === 'object' && raw !== null && 'type' in raw && raw.type === 'error') { + apply({ type: 'error', message: 'message' in raw ? String(raw.message ?? '') : '' }) + return + } + if (isSubscribeEvent(raw)) { + apply({ type: 'event', event: raw }) + } + }) return () => { - cancelled = true - unsubscribe() + endStream() void held .then(() => callAgentSession( diff --git a/mobile/src/transport/rpc-client-stream-registry.test.ts b/mobile/src/transport/rpc-client-stream-registry.test.ts index 0e7d561be6e3..8491ca0336a9 100644 --- a/mobile/src/transport/rpc-client-stream-registry.test.ts +++ b/mobile/src/transport/rpc-client-stream-registry.test.ts @@ -79,6 +79,20 @@ describe('RpcClientStreamRegistry', () => { ]) }) + it('ends one transcript stream on dispose and leaves a sibling on the same socket (U-03)', () => { + const { registry, sent } = createRegistry() + const disposeFirst = registry.subscribe('agentSession.subscribe', { sessionId: 's1' }, () => {}) + registry.subscribe('agentSession.subscribe', { sessionId: 's2' }, () => {}) + const [first] = sent + + disposeFirst() + expect(sent.at(-1)).toMatchObject({ + method: 'agentSession.unsubscribe', + params: { sessionId: 's1', subscriptionId: first!.id } + }) + expect(sent.filter((request) => request.method === 'agentSession.unsubscribe')).toHaveLength(1) + }) + it('keeps a disposed browser tombstone until ready can be unsubscribed', () => { const { registry, sent } = createRegistry() const dispose = registry.subscribe('browser.screencast', { page: 'page-1' }, () => {}) diff --git a/mobile/src/transport/rpc-client-stream-registry.ts b/mobile/src/transport/rpc-client-stream-registry.ts index 68e50a6e6ddf..5c491c382a53 100644 --- a/mobile/src/transport/rpc-client-stream-registry.ts +++ b/mobile/src/transport/rpc-client-stream-registry.ts @@ -213,7 +213,7 @@ export class RpcClientStreamRegistry { this.sendRpc('terminal.unsubscribe', params) } } else { - const unsubscribe = buildStreamUnsubscribe(stream?.method, stream?.params) + const unsubscribe = buildStreamUnsubscribe(stream?.method, stream?.params, id) if (unsubscribe) { this.sendRpc(unsubscribe.method, unsubscribe.params) } diff --git a/mobile/src/transport/rpc-client-terminal-subscription.ts b/mobile/src/transport/rpc-client-terminal-subscription.ts index 405f7f1d9e00..2bed2ab0458f 100644 --- a/mobile/src/transport/rpc-client-terminal-subscription.ts +++ b/mobile/src/transport/rpc-client-terminal-subscription.ts @@ -53,6 +53,14 @@ export function buildStreamUnsubscribe( } : null } + if (method === 'agentSession.subscribe') { + const sessionId = (params as { sessionId?: unknown }).sessionId + // The host keys each transcript stream by its frame id; without it every stream of the + // session on this socket would end. + return typeof sessionId === 'string' && requestId + ? { method: 'agentSession.unsubscribe', params: { sessionId, subscriptionId: requestId } } + : null + } if (method === 'nativeChat.subscribe') { const subscriptionId = (params as { subscriptionId?: unknown }).subscriptionId if (typeof subscriptionId === 'string') { diff --git a/src/main/claude/claude-context-usage-unloaded-turn.test.ts b/src/main/claude/claude-context-usage-unloaded-turn.test.ts index 83d9248f6489..b562c839efdb 100644 --- a/src/main/claude/claude-context-usage-unloaded-turn.test.ts +++ b/src/main/claude/claude-context-usage-unloaded-turn.test.ts @@ -18,10 +18,8 @@ import { createTrackedJournalOpener } from '../native-chat/agent-session-journal import { readAgentSessionHistory } from '../native-chat/agent-session-wire/agent-session-history-page' import type { StructuredAgentSessionAdapter } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' -import { - readStructuredAgentSessionOptions, - type StructuredAgentSessionMutationContext -} from '../native-chat/agent-session-wire/structured-agent-session-host-mutations' +import type { StructuredAgentSessionMutationContext } from '../native-chat/agent-session-wire/structured-agent-session-host-mutations' +import { readStructuredAgentSessionOptions } from '../native-chat/agent-session-wire/structured-agent-session-options-read' import { assistantFrame, initFrame, @@ -85,6 +83,11 @@ function readOptions( journal: AgentSessionJournal, adapter: Partial ) { + const running = { + journal, + child: { fence: 1, generation: 'generation-1' }, + params: { provider: 'claude' } + } // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the options read touches only these members. const context = { deps: { @@ -95,7 +98,8 @@ function readOptions( store: { getRecord: () => undefined } }, serialize: (_sessionId: string, task: () => Promise) => task(), - requireSession: () => ({ journal, fence: 1 }) + openConversation: async () => running, + conversation: async () => running } as unknown as StructuredAgentSessionMutationContext return readStructuredAgentSessionOptions(context, SESSION) } diff --git a/src/main/claude/claude-structured-dispatch-test-support.ts b/src/main/claude/claude-structured-dispatch-test-support.ts index a309bd9d0b98..5af0f17243b5 100644 --- a/src/main/claude/claude-structured-dispatch-test-support.ts +++ b/src/main/claude/claude-structured-dispatch-test-support.ts @@ -5,7 +5,7 @@ import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): ClaudeSession { return { @@ -32,7 +32,7 @@ export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): C capabilities: [], events: undefined, translator: null, - startup: { ...createClaudeSessionStartupGate(), state: 'proven' } + startup: { ...createClaudeSessionStartup(), state: 'proven' } } } diff --git a/src/main/claude/claude-structured-dispatch.ts b/src/main/claude/claude-structured-dispatch.ts index d93a8ecab0a3..5a5b27965958 100644 --- a/src/main/claude/claude-structured-dispatch.ts +++ b/src/main/claude/claude-structured-dispatch.ts @@ -35,10 +35,8 @@ import { import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' import { claudeStartupFailureReason, - claudeStartupHoldsWrites, - failClaudeStartupGate, - holdClaudeStartupWrite -} from './claude-structured-session-startup-gate' + failClaudeStartup +} from './claude-structured-session-startup-state' const MAX_ACTIVE_DISPATCH_WAITERS = 64 @@ -226,7 +224,7 @@ export function settleCancelledClaudeDispatchWaiters( * Retired rather than dropped: their identities stay joinable, bounded by * `MAX_RETIRED_DISPATCH_WAITERS`. */ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { - failClaudeStartupGate(session, new Error('claude stream-json ended before startup completed')) + failClaudeStartup(session, new Error('claude stream-json ended before startup completed')) for (const waiter of session.dispatchWaiters.splice(0)) { retireWaiter(session, waiter) waiter.resolve(null) @@ -236,8 +234,7 @@ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { export async function dispatchClaudeTurn( session: ClaudeSession, input: { clientMessageId?: string; body: AgentJournalMessageItem; requestedAt?: number }, - beforeDispatch?: () => Promise, - onSettledLate?: ClaudeLateDispatchSettlement + beforeDispatch?: () => Promise ): Promise { let content: unknown[] try { @@ -276,14 +273,6 @@ export async function dispatchClaudeTurn( parent_tool_use_id: null, session_id: session.providerSessionId } - if (claudeStartupHoldsWrites(session)) { - return holdClaudeStartupWrite(session, { - message, - arm, - ...(beforeDispatch ? { beforeDispatch } : {}), - ...(onSettledLate ? { settleLate: onSettledLate } : {}) - }) - } const pending = { replay: beforeDispatch ? undefined : arm() } const authorize = beforeDispatch ? async () => { diff --git a/src/main/claude/claude-structured-options.test.ts b/src/main/claude/claude-structured-options.test.ts index e02647f36253..a92149a2cddf 100644 --- a/src/main/claude/claude-structured-options.test.ts +++ b/src/main/claude/claude-structured-options.test.ts @@ -11,7 +11,7 @@ import { import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' import { claudeStructuredSessionOptionsFrom, observeClaudeFastModeFacts, @@ -48,7 +48,7 @@ function sessionFor(setModel: ClaudeSession['connection']['setModel']): ClaudeSe capabilities: [], events: undefined, translator: null, - startup: { ...createClaudeSessionStartupGate(), state: 'proven' } + startup: { ...createClaudeSessionStartup(), state: 'proven' } } } diff --git a/src/main/claude/claude-structured-options.ts b/src/main/claude/claude-structured-options.ts index 0090cd2f039f..5a8d8466822a 100644 --- a/src/main/claude/claude-structured-options.ts +++ b/src/main/claude/claude-structured-options.ts @@ -41,6 +41,13 @@ export function restoredClaudeStructuredSessionOptions( ) } +/** The keys `setClaudeStructuredOption` writes; a pick made at rest is checked against these. */ +const CLAUDE_STRUCTURED_OPTION_KEYS = new Set(['model', 'permissionMode', 'effort', 'fastMode']) + +export function isClaudeStructuredOptionKey(key: string): boolean { + return CLAUDE_STRUCTURED_OPTION_KEYS.has(key) +} + /** A client's write; the startup restore writes through `setClaudeStructuredOption` directly. */ export function setClaudeStructuredSessionOption( session: ClaudeSession, diff --git a/src/main/claude/claude-structured-prompt-ownership.ts b/src/main/claude/claude-structured-prompt-ownership.ts index 885ea2dfa74c..864a99a0b770 100644 --- a/src/main/claude/claude-structured-prompt-ownership.ts +++ b/src/main/claude/claude-structured-prompt-ownership.ts @@ -15,11 +15,6 @@ import { buildClaudePromptReply } from './claude-structured-prompt-replies' import type { ClaudeSession } from './claude-structured-session-state' import type { ClaudePendingPrompt } from './claude-prompt-registry' import type { PermissionResult } from '@anthropic-ai/claude-agent-sdk' -import { - claudeStartupHoldsWrites, - rejectClaudeStartupWrites -} from './claude-structured-session-startup-gate' -import { DISPATCH_REJECTED_CANCELLED } from '../../shared/structured-agent-session-dispatch-rejection' /** Conservative user-facing window: below the 30s control deadline, trading * residual slow-pump risk for ensuring delivery bookkeeping cannot block Stop indefinitely. */ @@ -107,14 +102,9 @@ export async function cancelClaudeStructuredTurn(input: { const session = requireSession(sessions, request.sessionId) const acquisitionGeneration = session.acquisitionGeneration const prompt = request.prompt - // A held prompt was never written, so Stop withdraws it; the drain only writes what it still - // holds. Before startup lands nothing was written, so there is nothing to interrupt either. - let withdrewHeld = false - if (!prompt && claudeStartupHoldsWrites(session) && session.fence === request.fence) { - withdrewHeld = rejectClaudeStartupWrites(session, DISPATCH_REJECTED_CANCELLED) - if (session.startup.state === 'pending') { - return { cancelled: withdrewHeld } - } + // Before startup lands nothing was written, so there is nothing to interrupt. + if (!prompt && session.startup.state === 'pending') { + return { cancelled: false } } if (prompt && session.fence !== request.fence) { return { cancelled: false } @@ -197,7 +187,7 @@ export async function cancelClaudeStructuredTurn(input: { } else if (claim) { session.prompts.releaseClaim(claim) } - return withdrewHeld ? { ...result, cancelled: true } : result + return result } catch (error) { if (claim && !interruptConfirmed) { session.prompts.releaseClaim(claim) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index 4181dab43c7a..c117e3e863f1 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -70,7 +70,7 @@ function realAdapter( const acquire = adapter.acquire adapter.acquire = async (input) => { const acquisition = await acquire(input) - await adapter.drainStartup(input.identity.sessionId) + await adapter.awaitStarted(input.identity.sessionId) return acquisition } return adapter diff --git a/src/main/claude/claude-structured-session-acquisition.ts b/src/main/claude/claude-structured-session-acquisition.ts index b2efed691ee9..6728f960ef03 100644 --- a/src/main/claude/claude-structured-session-acquisition.ts +++ b/src/main/claude/claude-structured-session-acquisition.ts @@ -248,30 +248,34 @@ export async function acquireClaudeSession({ event() } }) - session.startup.settled = settleClaudeSessionStartup({ - session, - facts: readClaudeStartupFacts({ - connection, - initProof, - sessionId, - providerSessionId: launch.providerSessionId, - resumesTranscript: launch.resumesTranscript, - inputOptions: input.options, - requestTimeoutMs: deps.requestTimeoutMs, - emit - }), - isCurrent: () => sessions.get(sessionId) === session, - requestTimeoutMs: deps.requestTimeoutMs, - fault: (error) => callbacks.handleExit(sessionId, attempt, error), - onStarted: (options) => - emit({ - type: 'started', + // Whichever comes first: the start landing or faulting, or the child being ended. + session.startup.settled = Promise.race([ + session.startup.settled, + settleClaudeSessionStartup({ + session, + facts: readClaudeStartupFacts({ + connection, + initProof, sessionId, - fence: input.fence, - acquisitionGeneration: session.acquisitionGeneration, - ...options - }) - }) + providerSessionId: launch.providerSessionId, + resumesTranscript: launch.resumesTranscript, + inputOptions: input.options, + requestTimeoutMs: deps.requestTimeoutMs, + emit + }), + isCurrent: () => sessions.get(sessionId) === session, + requestTimeoutMs: deps.requestTimeoutMs, + fault: (error) => callbacks.handleExit(sessionId, attempt, error), + onStarted: (options) => + emit({ + type: 'started', + sessionId, + fence: input.fence, + acquisitionGeneration: session.acquisitionGeneration, + ...options + }) + }) + ]) // A child whose exit already reached `handleExit` is not handed over as live: the create // fails with the CLI's own diagnostic, as one that died before publish does. if (sessions.get(sessionId) !== session) { diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index b8cc5c1dec58..4c67da4fa269 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -12,7 +12,10 @@ import { acquireClaudeSession } from './claude-structured-session-acquisition' import { supportsClaudeStructuredLocation } from './claude-structured-location-support' import { setClaudeStructuredSessionOption } from './claude-structured-options' import { readClaudeStructuredSessionOptions } from './claude-structured-session-options' -import { claudeStartupSettledWithin } from './claude-structured-session-startup-gate' +import { + claudeStartupFailureReason, + claudeStartupSettledWithin +} from './claude-structured-session-startup-state' import { CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS } from './claude-agent-sdk-control-requests' import { ClaudeAcquisitionRegistry, @@ -115,9 +118,16 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda * apart without guessing at wall-clock. */ drainObservedExits = (): Promise => drainClaudeObservedExits(this.exits) - /** Resolves once a published session's startup has landed or faulted it. */ - drainStartup = (sessionId: string): Promise => - this.sessions.get(sessionId)?.startup.settled ?? Promise.resolve() + /** Resolves once a published session's startup has landed, faulted, or been ended by a close; + * with the reason when it did not land. */ + awaitStarted = async (sessionId: string): Promise => { + const session = this.sessions.get(sessionId) + if (!session) { + return + } + await session.startup.settled + return claudeStartupFailureReason(session) ?? undefined + } /** Restart reconciliation reads the transcript a resume replays; these maps track liveness. */ providerHistoryWindow: NonNullable = ( @@ -179,9 +189,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } dispatch: StructuredAgentSessionAdapter['dispatch'] = (input) => - dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch, (settlement) => - this.deps.onDispatchSettledLate?.({ sessionId: input.sessionId, ...settlement }) - ) + dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch) compact: NonNullable = (input) => compactClaudeSession(this.session(input.sessionId), this.compactions, input) @@ -236,7 +244,8 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda ) readOptions = (input: { sessionId: string; fence: number }) => readClaudeStructuredSessionOptions(this.session(input.sessionId), this.deps.requestTimeoutMs) - recordsContextUsage = (sessionId: string): boolean => this.sessions.has(sessionId) + // Provider-level: a session at rest still reports the usage its journal recorded. + recordsContextUsage = (): boolean => true readOptionRestoreFailures = (sessionId: string): readonly string[] => [ ...(this.sessions.get(sessionId)?.restoreSkippedOptions ?? []) diff --git a/src/main/claude/claude-structured-session-exit-lifecycle.ts b/src/main/claude/claude-structured-session-exit-lifecycle.ts index 4f39a8dc4679..d0bddaf0a113 100644 --- a/src/main/claude/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/claude-structured-session-exit-lifecycle.ts @@ -3,7 +3,7 @@ import { claudeRootExitObserved, settleClaudeExitedSession } from './claude-structured-session-close' -import { failClaudeStartupGate } from './claude-structured-session-startup-gate' +import { failClaudeStartup } from './claude-structured-session-startup-state' import type { ClaudeAcquisitionAttempt, ClaudeSession, @@ -32,7 +32,7 @@ export function observeClaudeSessionExit( return } lifecycle.sessions.delete(sessionId) - failClaudeStartupGate(session, error) + failClaudeStartup(session, error) // Re-enter the provider's close ladder before publishing lifecycle recovery. // An exit callback is root evidence only; the retained tree proof must run // before the host releases and reacquires this exact child. diff --git a/src/main/claude/claude-structured-session-publication.ts b/src/main/claude/claude-structured-session-publication.ts index d64273f9b8b7..f20bc0c79077 100644 --- a/src/main/claude/claude-structured-session-publication.ts +++ b/src/main/claude/claude-structured-session-publication.ts @@ -6,7 +6,7 @@ import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' /** The session as published at spawn: nothing the CLI reports at init is assumed yet. */ export function createClaudeSessionPublication(input: { @@ -67,7 +67,7 @@ export function createClaudeSessionPublication(input: { translator: input.translator, events: input.events, ...(input.unbindReadingControl ? { unbindReadingControl: input.unbindReadingControl } : {}), - startup: createClaudeSessionStartupGate() + startup: createClaudeSessionStartup() } } } diff --git a/src/main/claude/claude-structured-session-recovery.test.ts b/src/main/claude/claude-structured-session-recovery.test.ts index ef8e0c654695..8ddc29a2e676 100644 --- a/src/main/claude/claude-structured-session-recovery.test.ts +++ b/src/main/claude/claude-structured-session-recovery.test.ts @@ -338,7 +338,7 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { spawnToken: 'spawn-9', events: journalSink }) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') const first = claude.connections[0] const oldPrompt = invokeCanUseTool(first, 'Bash', 'permission-retained', 'tool-retained') const oldSession = ( diff --git a/src/main/claude/claude-structured-session-startup-gate.ts b/src/main/claude/claude-structured-session-startup-gate.ts deleted file mode 100644 index 0df789361915..000000000000 --- a/src/main/claude/claude-structured-session-startup-gate.ts +++ /dev/null @@ -1,172 +0,0 @@ -// A Claude session is published once its child is spawned, before the CLI has answered -// initialize. Prompts sent in that window are held here and written, in order, once startup -// lands (init facts read and saved options restored), so a first turn never runs under -// defaults the restore was about to replace. A held prompt was never written, so a startup -// that fails rejects it rather than leaving its delivery in doubt. - -import type { AgentSessionDispatchOutcome } from '../native-chat/agent-session-wire/structured-agent-session-adapter' -import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' -import { dispatchWriteFailureReason } from '../../shared/structured-agent-session-dispatch-rejection' -import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' -import { claudeUserMessageWasProvablyUnwritten } from './claude-agent-sdk-user-message-queue' -import { - forgetRetiredWaiter, - forgetWaiter, - retireWaiter -} from './claude-structured-dispatch-waiters' -import type { - ClaudeDispatchWaiter, - ClaudeLateDispatchOutcome, - ClaudeSession -} from './claude-structured-session-state' - -type ClaudeStartupHeldWrite = { - waiter: ClaudeDispatchWaiter - message: Record - settleLate?: (outcome: ClaudeLateDispatchOutcome) => void -} - -export type ClaudeSessionStartupGate = { - state: 'pending' | 'proven' | 'failed' - held: ClaudeStartupHeldWrite[] - /** Held prompts are still being written; later prompts must queue behind them. */ - draining: boolean - failure: Error | null - /** Resolves once startup has landed or faulted the session; never rejects. */ - settled: Promise -} - -export function createClaudeSessionStartupGate(): ClaudeSessionStartupGate { - return { state: 'pending', held: [], draining: false, failure: null, settled: Promise.resolve() } -} - -export function claudeStartupFailureReason(session: ClaudeSession): string | null { - return session.startup.state === 'failed' - ? providerStartupFailureRejection(session.startup.failure ?? undefined) - : null -} - -/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */ -export function claudeStartupSettledWithin( - session: ClaudeSession | undefined, - timeoutMs: number -): Promise { - if (session?.startup.state !== 'pending') { - return Promise.resolve() - } - let timer: ReturnType | undefined - return Promise.race([ - session.startup.settled, - new Promise((resolve) => { - timer = setTimeout(resolve, timeoutMs) - }) - ]).finally(() => clearTimeout(timer)) -} - -export function claudeStartupHoldsWrites(session: ClaudeSession): boolean { - return session.startup.state === 'pending' || session.startup.draining -} - -/** Admits a prompt while startup is pending; it is written when `openClaudeStartupGate` runs. */ -export async function holdClaudeStartupWrite( - session: ClaudeSession, - input: { - message: Record - arm: () => { waiter: ClaudeDispatchWaiter } - beforeDispatch?: () => Promise - settleLate?: (outcome: ClaudeLateDispatchOutcome) => void - } -): Promise { - if (input.beforeDispatch) { - try { - await input.beforeDispatch() - } catch (error) { - if (error instanceof AgentSessionPreDispatchError) { - throw error - } - return { state: 'rejected', reason: dispatchWriteFailureReason(error) } - } - } - // Startup may have failed while the admission barrier ran. - const failed = claudeStartupFailureReason(session) - if (failed) { - return { state: 'rejected', reason: failed } - } - const { waiter } = input.arm() - session.startup.held.push({ - waiter, - message: input.message, - ...(input.settleLate ? { settleLate: input.settleLate } : {}) - }) - // Startup finished writing what it held while the barrier ran; nothing else would drain this. - if (!claudeStartupHoldsWrites(session)) { - void drainClaudeStartupWrites(session) - } - return { state: 'admitted' } -} - -export async function openClaudeStartupGate(session: ClaudeSession): Promise { - const gate = session.startup - if (gate.state !== 'pending') { - return - } - gate.state = 'proven' - await drainClaudeStartupWrites(session) -} - -async function drainClaudeStartupWrites(session: ClaudeSession): Promise { - const gate = session.startup - gate.draining = true - try { - for (let held = gate.held.shift(); held; held = gate.held.shift()) { - await writeHeld(session, held) - } - } finally { - gate.draining = false - } -} - -async function writeHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite): Promise { - try { - await session.connection.send(held.message) - } catch (error) { - if (held.waiter.settledUuid) { - return - } - if (claudeUserMessageWasProvablyUnwritten(error)) { - rejectHeld(session, held, dispatchWriteFailureReason(error)) - return - } - // Possibly written: only a replay or the child's exit can settle it now. - retireWaiter(session, held.waiter) - held.waiter.resolve(null) - } -} - -function rejectHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite, reason: string): void { - forgetWaiter(session, held.waiter) - forgetRetiredWaiter(session, held.waiter) - held.waiter.resolve(null) - if (held.waiter.clientMessageId) { - held.settleLate?.({ clientMessageId: held.waiter.clientMessageId, state: 'rejected', reason }) - } -} - -/** Rejects every held prompt with `reason`; true when any was held. */ -export function rejectClaudeStartupWrites(session: ClaudeSession, reason: string): boolean { - const held = session.startup.held.splice(0) - for (const entry of held) { - rejectHeld(session, entry, reason) - } - return held.length > 0 -} - -/** Startup cannot land any more; nothing held was written, so all of it is rejected. */ -export function failClaudeStartupGate(session: ClaudeSession, error: Error): void { - const gate = session.startup - if (gate.state === 'pending') { - gate.state = 'failed' - gate.failure = error - } - rejectClaudeStartupWrites(session, providerStartupFailureRejection(error)) -} diff --git a/src/main/claude/claude-structured-session-startup-state.ts b/src/main/claude/claude-structured-session-startup-state.ts new file mode 100644 index 000000000000..1bda560377ad --- /dev/null +++ b/src/main/claude/claude-structured-session-startup-state.ts @@ -0,0 +1,58 @@ +// Where a Claude start stands. A session is published once its child is spawned, before the CLI +// has answered initialize. Nothing is written to it until startup lands (init facts read and saved +// options restored): the host's delivery loop waits on `settled` before it hands a message over, +// so a first turn never runs under defaults the restore was about to replace. + +import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' +import type { ClaudeSession } from './claude-structured-session-state' + +export type ClaudeSessionStartup = { + state: 'pending' | 'proven' | 'failed' + failure: Error | null + /** Resolves once startup has landed or faulted, or the child exited or was closed; never + * rejects. A close must end it: the delivery loop waits here, and a start Stop cut short + * would otherwise hold that loop forever. */ + settled: Promise + end: () => void +} + +export function createClaudeSessionStartup(): ClaudeSessionStartup { + let end: () => void = () => undefined + const ended = new Promise((resolve) => { + end = resolve + }) + return { state: 'pending', failure: null, settled: ended, end } +} + +export function claudeStartupFailureReason(session: ClaudeSession): string | null { + return session.startup.state === 'failed' + ? providerStartupFailureRejection(session.startup.failure ?? undefined) + : null +} + +/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */ +export function claudeStartupSettledWithin( + session: ClaudeSession | undefined, + timeoutMs: number +): Promise { + if (session?.startup.state !== 'pending') { + return Promise.resolve() + } + let timer: ReturnType | undefined + return Promise.race([ + session.startup.settled, + new Promise((resolve) => { + timer = setTimeout(resolve, timeoutMs) + }) + ]).finally(() => clearTimeout(timer)) +} + +/** Startup cannot land any more: the child exited, was closed, or its start faulted. */ +export function failClaudeStartup(session: ClaudeSession, error: Error): void { + const startup = session.startup + if (startup.state === 'pending') { + startup.state = 'failed' + startup.failure = error + } + startup.end() +} diff --git a/src/main/claude/claude-structured-session-startup.test.ts b/src/main/claude/claude-structured-session-startup.test.ts index 362f4d691d02..ceb9e9cdf6d6 100644 --- a/src/main/claude/claude-structured-session-startup.test.ts +++ b/src/main/claude/claude-structured-session-startup.test.ts @@ -55,7 +55,7 @@ describe('Claude structured session publishes before the CLI answers initialize' expect(adapter.readCommands('session-1')).toBeUndefined() await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') expect(events.find((event) => event.type === 'options')).toMatchObject({ models: [{ value: 'claude-sonnet' }] @@ -65,7 +65,7 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.closeAll() }) - it('reports `started` once saved options are restored, before any held prompt is written', async () => { + it('reports `started` once saved options are restored, having written no prompt of its own', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, initModel: 'claude-opus-9' }) const { adapter, events } = startingAdapter(claude) const order: string[] = [] @@ -74,11 +74,10 @@ describe('Claude structured session publishes before the CLI answers initialize' return undefined } await adapter.acquire({ ...ACQUIRE, options: { model: 'opus' } }) - await adapter.dispatch(PROMPT) expect(events.some((event) => event.type === 'started')).toBe(false) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') const startedAt = events.findIndex((event) => event.type === 'started') expect(events[startedAt]).toEqual({ @@ -90,9 +89,9 @@ describe('Claude structured session publishes before the CLI answers initialize' reportedOptions: expect.objectContaining({ model: 'opus' }), restoreSkippedOptions: [] }) - // The restore wrote the saved model before `started`, and the held prompt only after it. + // The restore wrote the saved model before `started`; no message waits inside the adapter. expect(order).toEqual(['set_model']) - expect(claude.connections[0].sent).toHaveLength(1) + expect(claude.connections[0].sent).toEqual([]) expect(events.slice(0, startedAt).some((event) => event.type === 'options')).toBe(true) await adapter.closeAll() }) @@ -133,52 +132,37 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.closeAll() }) - it('holds a prompt sent before init and writes it once startup lands', async () => { + // The host's delivery loop waits here before it hands a message over, so the adapter no longer + // holds prompts of its own: nothing is written until startup lands because nothing is sent. + it('resolves awaitStarted only once startup lands', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) const { adapter } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - - await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' }) - expect(claude.connections[0].sent).toEqual([]) - - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') - - expect(claude.connections[0].sent).toHaveLength(1) - expect(claude.connections[0].sent[0]).toMatchObject({ type: 'user' }) - await adapter.closeAll() - }) - - it('writes a prompt whose admission barrier was still running when startup landed', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter } = startingAdapter(claude) - await adapter.acquire(ACQUIRE) - let passBarrier = (): void => {} - const barrier = new Promise((resolve) => { - passBarrier = resolve + let started = false + const waited = adapter.awaitStarted('session-1').then(() => { + started = true }) - const dispatched = adapter.dispatch({ ...PROMPT, beforeDispatch: () => barrier }) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') - passBarrier() + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS - 1) + expect(started).toBe(false) + await vi.advanceTimersByTimeAsync(1) + await waited - await expect(dispatched).resolves.toEqual({ state: 'admitted' }) - expect(claude.connections[0].sent.filter((message) => message.type === 'user')).toHaveLength(1) + await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' }) + expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]) await adapter.closeAll() }) - it('ends the session with the exit reason when the CLI dies before init, and rejects held prompts', async () => { + it('ends the session with the exit reason when the CLI dies before init', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, exitBeforeInit: 'claude stream-json exited (code 1): stderr says no' }) - const { adapter, events, late } = startingAdapter(claude) + const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() expect(events.find((event) => event.type === 'ended')).toMatchObject({ @@ -186,9 +170,6 @@ describe('Claude structured session publishes before the CLI answers initialize' cause: 'unexpected-exit', startupUnproven: true }) - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) expect(claude.connections[0].sent).toEqual([]) expect(claude.connections[0].closeCount).toBe(1) }) @@ -203,7 +184,7 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.acquire(ACQUIRE) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() // A failed start is released on the same evidence a failed create is. @@ -217,7 +198,7 @@ describe('Claude structured session publishes before the CLI answers initialize' const claude = fakeClaude({ initAccount: { apiProvider: 'firstParty', tokenSource: 'none' } }) const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() expect(events.find((event) => event.type === 'ended')).toMatchObject({ @@ -226,82 +207,41 @@ describe('Claude structured session publishes before the CLI answers initialize' }) }) - it('closes a session stopped before init without faulting it or writing held prompts', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, events, late } = startingAdapter(claude) + // A Stop that closes a child still starting must end the wait the host's delivery loop is in, + // though initialize never answers; otherwise every later send joins a loop that never moves. + it('ends the wait on a start closed before init, without faulting it', async () => { + const claude = fakeClaude({ initDelayMs: 10 * SLOW_INIT_MS }) + const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) + let ended = false + const waited = adapter.awaitStarted('session-1').then(() => { + ended = true + }) await expect(adapter.closeSession('session-1')).resolves.toBe(true) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await vi.advanceTimersByTimeAsync(0) + await waited + expect(ended).toBe(true) const connection = claude.connections[0] expect(connection.closeCount).toBe(1) expect(connection.sent).toEqual([]) expect(connection.calls.map(({ subtype }) => subtype)).not.toContain('get_settings') - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) expect(events.some((event) => event.type === 'ended' && event.startupUnproven)).toBe(false) expect(events.some((event) => event.type === 'started')).toBe(false) }) - it('withdraws a held prompt when the turn is cancelled before init', async () => { + it('interrupts nothing when Stop lands before init: nothing was written', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, late } = startingAdapter(claude) + const { adapter } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) await expect( adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) - ).resolves.toEqual({ cancelled: true }) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + ).resolves.toEqual({ cancelled: false }) + expect(claude.connections[0].calls.map(({ subtype }) => subtype)).not.toContain('interrupt') expect(claude.connections[0].sent).toEqual([]) - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) - await adapter.closeAll() - }) - - it('withdraws the prompts still held when Stop lands while startup is writing them', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, late } = startingAdapter(claude) - await adapter.acquire(ACQUIRE) - const connection = claude.connections[0] - const send = connection.send - let landFirstWrite = (): void => {} - const firstWrite = new Promise((resolve) => { - landFirstWrite = resolve - }) - let writes = 0 - connection.send = async (message, beforeDispatch) => { - writes += 1 - if (writes === 1) { - await firstWrite - } - return send(message, beforeDispatch) - } - await adapter.dispatch(PROMPT) - await adapter.dispatch({ ...PROMPT, clientMessageId: 'client-2' }) - - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - // Startup has landed and is writing the first held prompt. - expect(writes).toBe(1) - const cancelled = adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) - await vi.advanceTimersByTimeAsync(0) - landFirstWrite() - await vi.advanceTimersByTimeAsync(5_000) - await adapter.drainStartup('session-1') - - expect(connection.sent.filter((message) => message.type === 'user')).toHaveLength(1) - expect(late).toContainEqual( - expect.objectContaining({ clientMessageId: 'client-2', state: 'rejected' }) - ) - // Stop withdrew something, so it answers as a cancel whatever the interrupt made of the turn. - await expect(cancelled).resolves.toEqual({ cancelled: true }) await adapter.closeAll() }) }) diff --git a/src/main/claude/claude-structured-session-startup.ts b/src/main/claude/claude-structured-session-startup.ts index aa1f9a02eccd..d486a670a13e 100644 --- a/src/main/claude/claude-structured-session-startup.ts +++ b/src/main/claude/claude-structured-session-startup.ts @@ -26,10 +26,7 @@ import { observeClaudeSettingsApplied, readClaudeSettingsEffort } from './claude-structured-session-options' -import { - failClaudeStartupGate, - openClaudeStartupGate -} from './claude-structured-session-startup-gate' +import { failClaudeStartup } from './claude-structured-session-startup-state' import type { ClaudeSession, ClaudeStructuredSessionEvent } from './claude-structured-session-state' export type ClaudeInitProof = { @@ -163,8 +160,8 @@ function claudeStartedReportedOptions( return persisted } -/** Applies startup facts to the published session, restores saved options, then releases - * held prompts. Any failure faults the session so the user sees why it never started. */ +/** Applies startup facts to the published session and restores saved options; only then does the + * session take input. Any failure faults the session so the user sees why it never started. */ export async function settleClaudeSessionStartup(input: { session: ClaudeSession facts: Promise @@ -179,7 +176,7 @@ export async function settleClaudeSessionStartup(input: { if (input.isCurrent()) { return false } - failClaudeStartupGate(session, new Error('claude session closed before startup completed')) + failClaudeStartup(session, new Error('claude session closed before startup completed')) return true } try { @@ -198,13 +195,15 @@ export async function settleClaudeSessionStartup(input: { ), restoreSkippedOptions: [...session.restoreSkippedOptions] }) - await openClaudeStartupGate(session) + if (session.startup.state === 'pending') { + session.startup.state = 'proven' + } } } catch (caught) { const error = caught instanceof Error ? caught : new Error(String(caught)) // A close or exit that already ended startup owns how the session ends. const endedElsewhere = session.startup.state !== 'pending' - failClaudeStartupGate(session, error) + failClaudeStartup(session, error) if (!endedElsewhere && input.isCurrent()) { input.fault(error) } diff --git a/src/main/claude/claude-structured-session-state.ts b/src/main/claude/claude-structured-session-state.ts index 37cc762c6f5e..32f04b88350f 100644 --- a/src/main/claude/claude-structured-session-state.ts +++ b/src/main/claude/claude-structured-session-state.ts @@ -25,7 +25,7 @@ import type { AgentChildWorkEvidence } from '../../shared/agent-status-child-wor import type { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import type { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import type { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import type { ClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import type { ClaudeSessionStartup } from './claude-structured-session-startup-state' export type ClaudeAuthDiagnostic = { apiKeySourceConfigured: boolean @@ -196,7 +196,7 @@ export type ClaudeSession = { events: StructuredAgentSessionEventSink | undefined unbindReadingControl?: () => void /** Published at spawn; init facts, option restore and queued prompts land when startup does. */ - startup: ClaudeSessionStartupGate + startup: ClaudeSessionStartup } export function mintClaudeAcquisitionGeneration(deps: ClaudeStructuredSessionAdapterDeps): string { diff --git a/src/main/claude/claude-structured-session-test-support.ts b/src/main/claude/claude-structured-session-test-support.ts index 023e0ba46b70..c8e3d86b0e86 100644 --- a/src/main/claude/claude-structured-session-test-support.ts +++ b/src/main/claude/claude-structured-session-test-support.ts @@ -220,7 +220,7 @@ export function adapterFor( const acquire = adapter.acquire adapter.acquire = async (input) => { const acquisition = await acquire(input) - await adapter.drainStartup(input.identity.sessionId) + await adapter.awaitStarted(input.identity.sessionId) return acquisition } return adapter diff --git a/src/main/codex/codex-structured-session-adapter.ts b/src/main/codex/codex-structured-session-adapter.ts index b9affd8bd003..8cb066511e93 100644 --- a/src/main/codex/codex-structured-session-adapter.ts +++ b/src/main/codex/codex-structured-session-adapter.ts @@ -277,7 +277,8 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap this.deps.requestTimeoutMs ) - supportsThreadGoal = (sessionId: string): boolean => this.sessions.has(sessionId) + // Provider-level: a goal change at rest starts the agent first. + supportsThreadGoal = (): boolean => true answerPrompt: StructuredAgentSessionAdapter['answerPrompt'] = (request) => answerCodexStructuredPrompt({ request, sessions: this.sessions }) diff --git a/src/main/ipc/desktop-renderer-runtime-capabilities.ts b/src/main/ipc/desktop-renderer-runtime-capabilities.ts index 63ba9b1d0b11..0bcfd1bd686d 100644 --- a/src/main/ipc/desktop-renderer-runtime-capabilities.ts +++ b/src/main/ipc/desktop-renderer-runtime-capabilities.ts @@ -1,5 +1,6 @@ import { AGENT_LAUNCH_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, @@ -25,6 +26,7 @@ import { export const DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES: readonly RuntimeCapability[] = [ AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_TURN_ITEM_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, diff --git a/src/main/native-chat/agent-session-journal/journal-cursor-probe.test.ts b/src/main/native-chat/agent-session-journal/journal-cursor-probe.test.ts new file mode 100644 index 000000000000..e9bce1a81027 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-cursor-probe.test.ts @@ -0,0 +1,118 @@ +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { probeJournalCursor } from './journal-cursor-probe' +import { journalDatabaseFile } from './journal-paths' +import { openAgentSessionJournal } from './journal-store-factory' + +const SESSION = 'session-alpha' +let dirs: string[] = [] + +afterEach(() => { + dirs.forEach((dir) => rmSync(dir, { recursive: true, force: true })) + dirs = [] +}) + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'orca-journal-probe-')) + dirs.push(dir) + return dir +} + +/** A closed, checkpointed journal holding one message, and the position its open computes. */ +async function settledJournal(journalDir: string) { + const journal = await openAgentSessionJournal({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } + }, + journalDir + }) + await journal.appendItem( + { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 0 }, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hello' }] }, + { fence: 1 } + ) + const cursor = journal.cursor() + await journal.close() + return cursor +} + +describe('journal cursor probe', () => { + it('reads the position the open computes and leaves the database file byte-identical (C13)', async () => { + const journalDir = tempDir() + const cursor = await settledJournal(journalDir) + const dbPath = journalDatabaseFile(journalDir) + const sidecars = [`${dbPath}-wal`, `${dbPath}-shm`] + // As a cleanly closed journal leaves it: the main file alone. Its mode loosened, as a copied or + // restored profile can leave it, because SQLite gives new sidecars the main file's mode. + sidecars.forEach((path) => rmSync(path, { force: true })) + if (process.platform !== 'win32') { + chmodSync(dbPath, 0o644) + } + const before = readFileSync(dbPath) + + expect(probeJournalCursor(journalDir, SESSION)).toEqual(cursor) + + expect(readFileSync(dbPath).equals(before)).toBe(true) + if (process.platform !== 'win32') { + // A read-only open of a WAL file leaves its sidecars behind; they stay owner-only. + expect(sidecars.filter((path) => existsSync(path))).not.toEqual([]) + for (const sidecar of sidecars.filter((path) => existsSync(path))) { + expect(statSync(sidecar).mode & 0o777).toBe(0o600) + } + } + }) + + it('reads a turn a crash left only in the write-ahead log (C15)', async () => { + const journalDir = tempDir() + const settled = await settledJournal(journalDir) + const dbPath = journalDatabaseFile(journalDir) + // A later turn commits into the WAL, and the process dies before any checkpoint. + const writer = new Database(dbPath) + writer.pragma('wal_autocheckpoint = 0') + writer.exec('BEGIN IMMEDIATE') + const insert = writer.prepare( + 'INSERT INTO journal_rows (session_id, epoch, seq, ts, row_json) VALUES (?, ?, ?, ?, ?)' + ) + for (let offset = 1; offset <= 5; offset += 1) { + insert.run(SESSION, settled.epoch, settled.sequence + offset, 1, '{}') + } + writer.exec('COMMIT') + const crashed = join(tempDir(), 'journal') + mkdirSync(crashed) + copyFileSync(dbPath, journalDatabaseFile(crashed)) + copyFileSync(`${dbPath}-wal`, `${journalDatabaseFile(crashed)}-wal`) + writer.close() + + expect(probeJournalCursor(crashed, SESSION)).toEqual({ + epoch: settled.epoch, + sequence: settled.sequence + 5 + }) + }) + + it('misses, rather than throws, on a file that is not a database', () => { + const journalDir = tempDir() + mkdirSync(journalDir, { recursive: true }) + const dbPath = journalDatabaseFile(journalDir) + rmSync(dbPath, { force: true }) + expect(probeJournalCursor(journalDir, SESSION)).toBeNull() + writeFileSync(dbPath, Buffer.alloc(8192, 7)) + expect(probeJournalCursor(journalDir, SESSION)).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-cursor-probe.ts b/src/main/native-chat/agent-session-journal/journal-cursor-probe.ts new file mode 100644 index 000000000000..1260113768f8 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-cursor-probe.ts @@ -0,0 +1,56 @@ +// Where a journal stands, read without opening it for use. +// +// A restarted host lists a chat from its saved status only while the journal is still exactly where +// that status was computed. This reads the position the open would compute — the live epoch and its +// newest row — on a read-only connection that sets no pragma and runs no DDL, so the file is left +// byte-identical. Any doubt is a miss (null): the caller then opens the journal properly. +// +// Never `immutable` or `nolock`: those skip the `-wal`, and a turn a crash left only in the WAL +// would read as the old position and bring back a stale "done". + +import { existsSync } from 'node:fs' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import { hardenSqliteDatabaseFiles } from '../../sqlite/harden-database-files' +import Database from '../../sqlite/sync-database' +import { journalPragmaNumber } from './journal-database' +import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' +import { journalDatabaseFile } from './journal-paths' +import { pendingJournalRepairSequence } from './journal-repair-marker' +import { readJournalSessionEpoch } from './journal-row-table' + +const SELECT_NEWEST_SEQUENCE = + 'SELECT seq FROM journal_rows WHERE session_id = ? AND epoch = ? ORDER BY seq DESC LIMIT 1' + +export function probeJournalCursor( + journalDir: string, + sessionId: string +): AgentJournalCursor | null { + const dbPath = journalDatabaseFile(journalDir) + if (!existsSync(dbPath)) { + return null + } + let db: Database.Database | null = null + try { + db = new Database(dbPath, { readonly: true, fileMustExist: true }) + if (journalPragmaNumber(db, 'user_version') !== JOURNAL_DB_SCHEMA_VERSION) { + return null + } + const epoch = readJournalSessionEpoch(db, sessionId) + if (!epoch || pendingJournalRepairSequence(db, sessionId, epoch) !== null) { + return null + } + const newest = db.prepare(SELECT_NEWEST_SEQUENCE).get(sessionId, epoch) + const sequence: unknown = newest?.seq + return typeof sequence === 'number' ? { epoch, sequence } : null + } catch { + return null + } finally { + try { + db?.close() + } catch { + // A close that fails leaves nothing this probe can use; the answer stands. + } + // A read-only open of a WAL file leaves `-wal`/`-shm` behind at umask permissions. + hardenSqliteDatabaseFiles(dbPath) + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index c4138ba22418..c356d7904b23 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -1,8 +1,11 @@ +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { DISPATCH_DOUBT_HOST_RESTARTED } from './journal-dispatch-doubt-reasons' import type { AgentSessionJournal } from './journal-store' /** Settles every submission a process fact left unanswerable. Doubt is never - * proof of non-delivery, so nothing here ever becomes re-deliverable. */ + * proof of non-delivery, so nothing here ever becomes re-deliverable. A queued + * submission was never handed over, so it is not in doubt and is left alone. */ export async function markJournalPendingSubmissionsUnknown( journal: AgentSessionJournal, fence: number, @@ -12,8 +15,9 @@ export async function markJournalPendingSubmissionsUnknown( .submissions() .filter( (entry) => - entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true) + !isQueuedAgentJournalSubmission(entry) && + (entry.dispatchState === 'pending' || + (entry.dispatchState === 'unknown' && entry.recovered !== true)) ) for (const entry of unresolved) { // An earlier reason already names a sharper fact than "the host restarted". @@ -31,7 +35,8 @@ export async function markJournalPendingSubmissionsUnknown( } /** Settles every submission a child that never proved its start left unanswered as `rejected`: - * such a child accepted nothing, so each is provably unwritten and safe to send again. */ + * such a child accepted nothing, so each is provably unwritten and safe to send again. A queued + * submission was never handed to that child; the delivery loop settles it. */ export async function rejectJournalPendingSubmissions( journal: AgentSessionJournal, fence: number, @@ -41,8 +46,9 @@ export async function rejectJournalPendingSubmissions( .submissions() .filter( (entry) => - entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true) + !isQueuedAgentJournalSubmission(entry) && + (entry.dispatchState === 'pending' || + (entry.dispatchState === 'unknown' && entry.recovered !== true)) ) for (const entry of unwritten) { await journal.resolveDispatch({ @@ -55,3 +61,25 @@ export async function rejectJournalPendingSubmissions( } return unwritten.map((entry) => entry.clientMessageId) } + +/** Rejects queued submissions — accepted, never handed over, so provably unwritten. */ +export async function rejectJournalQueuedSubmissions( + journal: AgentSessionJournal, + fence: number, + reason: string, + which: (submission: AgentJournalSubmission) => boolean = () => true +): Promise { + const queued = journal + .submissions() + .filter((entry) => isQueuedAgentJournalSubmission(entry) && which(entry)) + for (const entry of queued) { + await journal.resolveDispatch({ + clientMessageId: entry.clientMessageId, + state: 'rejected', + reason, + fence, + recovered: true + }) + } + return queued.map((entry) => entry.clientMessageId) +} diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 273502c8cf2b..3fbee85a417e 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -254,7 +254,8 @@ function applySubmission( providerItemId: null, reason: null, submittedAt: row.ts, - resolvedAt: null + resolvedAt: null, + ...(row.handoverRecorded ? { handoverRecorded: true, acceptedSequence: row.seq } : {}) }) const itemId = agentJournalSubmissionKey(row.clientMessageId) upsertItem(state, itemId, 0, journalRenderItem(itemId, 0, row.body, row)) @@ -277,6 +278,9 @@ function applyDispatch( submission.providerItemId = row.providerItemId submission.reason = row.reason submission.resolvedAt = row.state === 'pending' ? null : row.ts + if (row.state === 'pending') { + submission.handedOverAt = row.ts + } if (row.recovered) { submission.recovered = row.recovered } else { diff --git a/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts b/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts index 626baa9820aa..f37571814bbf 100644 --- a/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts +++ b/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts @@ -18,6 +18,7 @@ import { agentJournalItemKey, agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionJournal } from './journal-store' import { reconcileSubmissions, type ProviderHistoryWindow } from './journal-submission-reconciler' @@ -42,7 +43,11 @@ function comparableSubmissions(journal: AgentSessionJournal): AgentJournalSubmis const { items, submissions } = journal.snapshot() const bodies = new Map(items.map((item) => [item.itemId, item.body])) return submissions.filter((submission) => { - if (submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') { + if ( + (submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') || + // Never handed over, so provider history cannot hold it. + isQueuedAgentJournalSubmission(submission) + ) { return false } const body = bodies.get(agentJournalSubmissionKey(submission.clientMessageId)) diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index e17ccf0cdf87..e56ca86a7fb5 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -65,6 +65,7 @@ export function journalSubmissionRowBuilder( payloadFingerprint: string body: AgentJournalMessageItem fence: number + handoverRecorded?: true } ): RowBuilder { return (seq, ts) => @@ -267,6 +268,7 @@ export function buildJournalSubmissionRow(input: { seq: number fence: number ts: number + handoverRecorded?: true }): JournalSubmissionRow { return { kind: 'submission', @@ -274,7 +276,8 @@ export function buildJournalSubmissionRow(input: { payloadFingerprint: input.payloadFingerprint, providerHandle: input.providerHandle, body: input.body, - ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts) + ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts), + ...(input.handoverRecorded ? { handoverRecorded: true } : {}) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index 99cb47971e95..0f29f8b38794 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -77,6 +77,9 @@ export type JournalSubmissionRow = JournalRowBase & { payloadFingerprint: string providerHandle: AgentSessionProviderHandle body: AgentJournalMessageItem + /** Accepted to be handed over by a later `dispatch{pending}` row; absent on rows whose writer + * dispatched in the same step. Older readers keep the key and ignore it. */ + handoverRecorded?: true } export type JournalDispatchRow = JournalRowBase & { diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index cea47e381fe4..79d86f80dbbf 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -59,6 +59,8 @@ export type JournalSubmissionInput = { payloadFingerprint: string body: AgentJournalMessageItem fence: number + /** The send is accepted now and handed over later, by a `dispatch{pending}` row. */ + handoverRecorded?: true } export type JournalItemAppendInput = { diff --git a/src/main/native-chat/agent-session-journal/journal-store.test.ts b/src/main/native-chat/agent-session-journal/journal-store.test.ts index ffaea5c5d6f1..3637f03dab6c 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.test.ts @@ -424,3 +424,31 @@ async function withJournalDatabase( opened.db.close() } } + +describe('what a handle found on disk when it opened', () => { + const submission = (clientMessageId: string) => ({ + clientMessageId, + payloadFingerprint: 'fp', + body: { kind: 'message' as const, role: 'user' as const, blocks: [] }, + fence: 1, + handoverRecorded: true as const + }) + + it('names rows an earlier handle wrote, and never a row of a later epoch', async () => { + const earlier = await open() + await earlier.appendItem(item(1), body('one'), { fence: 1 }) + await earlier.appendSubmission(submission('earlier')) + await earlier.close() + + const journal = await open() + const leftover = journal.submissions().find((entry) => entry.clientMessageId === 'earlier') + expect(journal.wroteBeforeOpen(leftover?.acceptedSequence)).toBe(true) + + // Sequences restart with an epoch, so a row accepted after it can sit below the open cursor. + await journal.replaceEpochItems('handle_forked', 1, []) + await journal.appendSubmission(submission('later')) + const later = journal.submissions().find((entry) => entry.clientMessageId === 'later') + expect(later?.acceptedSequence).toBeLessThanOrEqual(2) + expect(journal.wroteBeforeOpen(later?.acceptedSequence)).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 568fbe4d39d3..5119f7ef597c 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -28,7 +28,8 @@ import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open' import { journalDatabaseFile } from './journal-paths' import { markJournalPendingSubmissionsUnknown, - rejectJournalPendingSubmissions + rejectJournalPendingSubmissions, + rejectJournalQueuedSubmissions } from './journal-pending-submission-recovery' import { applyJournalRow, @@ -75,6 +76,7 @@ export class AgentSessionJournal { private state: JournalReducerState private readOnly = false private malformedRows = 0 + private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } private database: OpenJournalDatabase | null = null private onCommitted: (() => void) | null = null private readonly queue: JournalWriteQueue @@ -147,6 +149,16 @@ export class AgentSessionJournal { return this.journalDir } + /** Whether a row at this sequence was on disk when this handle opened, so an earlier handle + * wrote it. Sequences restart with each epoch, so a row of a later epoch never was. */ + wroteBeforeOpen(sequence: number | undefined): boolean { + return ( + sequence !== undefined && + this.state.epoch === this.openedThrough.epoch && + sequence <= this.openedThrough.sequence + ) + } + /** What the last open's repair did. */ get repair(): { malformedRows: number } { return { malformedRows: this.malformedRows } @@ -157,6 +169,7 @@ export class AgentSessionJournal { this.database = openJournalDatabase(this.dbPath) try { await this.restore() + this.openedThrough = this.cursor() } catch (error) { // Nothing else holds a reference to this connection, so a throw here is // the leak site unless the store releases it itself — and a close that @@ -310,6 +323,15 @@ export class AgentSessionJournal { return rejectJournalPendingSubmissions(this, fence, reason) } + /** Reject sends accepted but never handed over, optionally only those `which` names. */ + async rejectQueuedSubmissions( + fence: number, + reason: string, + which?: (submission: AgentJournalSubmission) => boolean + ): Promise { + return rejectJournalQueuedSubmissions(this, fence, reason, which) + } + /** The escape hatch for corruption, an unreconcilable prefix, a forked handle, * and an unreadable schema. It invalidates every cursor; clients reload. */ async rollEpoch(reason: AgentJournalEpochReason, fence: number): Promise { diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts index 214c889b5c9d..1c60029c7d32 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts @@ -6,14 +6,14 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentJournalItemIdentity, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { openJournalDatabase } from '../agent-session-journal/journal-database' import { JOURNAL_DB_SCHEMA_VERSION } from '../agent-session-journal/journal-database-schema' -import { loadJournal } from '../agent-session-journal/journal-open' +import { loadJournal, replayJournal } from '../agent-session-journal/journal-open' import { journalDatabaseFile } from '../agent-session-journal/journal-paths' import { readJournalEpochRows } from '../agent-session-journal/journal-row-table' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' @@ -24,6 +24,12 @@ import { recoveryJournalDir } from './agent-session-journal-recovery' +// Only the store's own replay goes through the mock; the probe's, inside the same module, does not. +vi.mock('../agent-session-journal/journal-open', async (importOriginal) => { + const actual = await importOriginal<{ replayJournal: typeof replayJournal }>() + return { ...actual, replayJournal: vi.fn(actual.replayJournal) } +}) + const CODEX_SESSION = '019fd532-7c11-7a90-b6de-4e1a2c3d5f60' const IDENTITY: AgentSessionJournalIdentity = { @@ -160,6 +166,21 @@ describe('openAgentSessionJournalWithRecovery', () => { expect(opened.snapshot().items).toHaveLength(2) }) + it('reads the journal once: the probe is the open', async () => { + await seedJournal(2) + vi.mocked(replayJournal).mockClear() + const opened = journals.track( + await openAgentSessionJournalWithRecovery({ + identity: IDENTITY, + journalDir, + fence: 1, + historyFilePath + }).then((result) => result.journal) + ) + expect(opened.snapshot().items).toHaveLength(2) + expect(replayJournal).not.toHaveBeenCalled() + }) + it('rebuilds a holed journal in place on a fresh epoch', async () => { await seedJournal(3) await deleteRow(3) diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts index 6e771a318092..29d324be9718 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts @@ -67,7 +67,9 @@ export async function openAgentSessionJournalWithRecovery(input: { } const journal = await openAgentSessionJournal({ identity: input.identity, - journalDir: input.journalDir + journalDir: input.journalDir, + // The probe is this open's replay; omitted, not `null`, when there was nothing to load. + ...(probe ? { loaded: probe } : {}) }) if (!probe?.corrupt) { return { journal, recovery: null } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts new file mode 100644 index 000000000000..b0d658178e04 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -0,0 +1,674 @@ +// A send is accepted, then delivered: the host answers once the message is recorded, and the +// session's delivery loop starts a provider child for it and hands it over. Against the real host, +// store and journal; each assertion reads what an open chat or the journal's next reader sees. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { + DISPATCH_REJECTED_CANCELLED, + DISPATCH_REJECTED_HOST_RESTARTED, + DISPATCH_REJECTED_PROVIDER_CLOSED +} from '../../../shared/structured-agent-session-dispatch-rejection' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { + AgentSessionPreSpawnError, + type StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' +import { journalIdentityFor } from './structured-agent-session-attach' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { persistRewindRecord } from './structured-rewind-recovery' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let dispatch: Mock +let adapterExtras: Partial +let idleMs: number + +const spawnChild: StructuredAgentSessionAdapter['acquire'] = async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + acquisitionGeneration: `generation-${acquire.mock.calls.length}`, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } +}) + +async function startHost(): Promise { + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: vi.fn(async () => ({ cancelled: false })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined), + ...adapterExtras + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + idleSweep: { intervalMs: 5, idleMs }, + now: () => NOW + }) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-accept-deliver-')) + resetHostTestOperationIds() + adapterExtras = {} + idleMs = 60 * 60_000 + acquire = vi.fn(spawnChild) + dispatch = vi.fn(async () => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD, + turnId: `turn-${dispatch.mock.calls.length}`, + ordinal: dispatch.mock.calls.length + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + await startHost() + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function sendParams(text: string) { + const body = hostTestMessage(text) + return { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } +} + +/** Accepted at once, whatever the child is doing; answers the message id. */ +async function accept(text: string): Promise { + const params = sendParams(text) + expect(await host.send(CALLER, params)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } + }) + return params.envelope.clientOperationId +} + +function stop() { + const turnId = 'turn-none' + return host.cancel(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: { turnId } + }) + }, + turnId + }) +} + +async function submission(id: string): Promise { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === id + ) +} + +/** Read back after the conversation was closed, through the same open any reader takes. */ +async function reopened(id: string): Promise { + await host.revealSession(SESSION) + return submission(id) +} + +async function errorRows(): Promise { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) +} + +let subscriptions = 0 + +async function subscribe(): Promise { + const events: AgentSessionSubscribeEvent[] = [] + subscriptions += 1 + // Cloned as received: a frame shares the journal's live objects, which later rows revise. + await host.subscribe({ + id: `sub-${subscriptions}`, + sessionId: SESSION, + emit: (event) => events.push(structuredClone(event)) + }) + return events +} + +/** What an open chat was told about one message, in frame order. */ +function framedStates(events: AgentSessionSubscribeEvent[], id: string): string[] { + return events.flatMap((event) => + event.type === 'batch' + ? event.batch.submissions + .filter((entry) => entry.clientMessageId === id) + .map((entry) => + entry.dispatchState === 'pending' && entry.handedOverAt !== undefined + ? 'handed-over' + : entry.dispatchState + ) + : [] + ) +} + +function deferred() { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((next, fail) => { + resolve = next + reject = fail + }) + return { promise, resolve, reject } +} + +/** Rows written by an earlier host process that ended before handing them over. */ +async function writeAsEarlierProcess( + write: (journal: AgentSessionJournal, fence: number) => Promise +): Promise { + await host.close(SESSION) + const record = store.getRecord(SESSION)! + const params = attachParamsForRecord(record, { + clientOperationId: 'earlier', + expectedRuntimeFence: record.lease.runtimeFence + }) + const journal = await openAgentSessionJournal({ + identity: journalIdentityFor(record, params), + journalDir: journalDirectoryFor(root, { + workspaceId: record.location.workspaceId, + sessionId: SESSION + }) + }) + await write(journal, record.lease.runtimeFence) + await journal.close() +} + +function earlierSubmission(id: string, text: string, handoverRecorded?: true) { + const body = hostTestMessage(text) + return { + clientMessageId: id, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }), + body, + ...(handoverRecorded ? { handoverRecorded } : {}) + } +} + +describe('a send is answered at acceptance', () => { + it('answers before the child starts, then an open chat sees the handover and the reply (W2)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + + const answering = deferred() + const answer = dispatch.getMockImplementation()! + dispatch.mockImplementationOnce(async (input) => { + await answering.promise + return answer(input) + }) + + const id = await accept('hello') + const events = await subscribe() + await eventually(async () => expect(acquire).toHaveBeenCalledTimes(2)) + expect(dispatch).not.toHaveBeenCalled() + + starting.resolve() + await eventually(async () => expect(framedStates(events, id)).toEqual(['handed-over'])) + answering.resolve() + await eventually(async () => + expect(framedStates(events, id)).toEqual(['handed-over', 'accepted']) + ) + }) + + it('accepts a second send while the first one starts the child, before handing either over (W6)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + const first = await accept('first') + await eventually(async () => expect(acquire).toHaveBeenCalledTimes(2)) + const second = host.send(CALLER, sendParams('second')) + + starting.resolve() + const secondResult = await second + if (!secondResult.ok) { + throw new Error('the second send was refused') + } + const secondId = secondResult.value.clientMessageId + await eventually(async () => + expect((await submission(secondId))?.dispatchState).toBe('accepted') + ) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([first, secondId]) + // The second was accepted while the start held the queue — before the first was handed over. + expect((await submission(secondId))!.submittedAt).toBeLessThanOrEqual( + (await submission(first))!.handedOverAt! + ) + const rows = (await host.journalSnapshot(SESSION)).submissions + expect(rows.map((row) => row.clientMessageId)).toEqual([first, secondId]) + }) +}) + +describe('a start the chat needed and did not get', () => { + it('writes one error row and rejects every queued message with it; the next send starts (W3)', async () => { + await host.close(SESSION) + acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT')) + const first = await accept('first') + const second = await accept('second') + + await eventually(async () => expect((await submission(second))?.dispatchState).toBe('rejected')) + const rows = await errorRows() + expect(rows).toHaveLength(1) + expect(rows[0]).toContain('spawn codex ENOENT') + expect(await submission(first)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] }) + expect(await submission(second)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] }) + + const next = await accept('after the fix') + await eventually(async () => expect((await submission(next))?.dispatchState).toBe('accepted')) + expect(await errorRows()).toHaveLength(1) + }) + + it.each([ + [ + 'eligibility', + () => { + adapterExtras = { supportsLocation: () => false } + }, + 'cannot resume' + ], + [ + 'spawn', + () => acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT')), + 'spawn codex ENOENT' + ], + [ + 'auth', + () => + acquire.mockRejectedValueOnce( + new AgentSessionPreSpawnError(new Error('Not logged in. Please run /login.')) + ), + 'Not logged in' + ] + ])('writes one row a live chat sees for a %s refusal (W14)', async (_source, arrange, cause) => { + await host.close(SESSION) + arrange() + await host.flushAllStreamedEvents() + await startHost() + const id = await accept('hello') + const events = await subscribe() + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect(await errorRows()).toHaveLength(1) + expect((await errorRows())[0]).toContain(cause) + const framedRows = events.flatMap((event) => + event.type === 'batch' || event.type === 'snapshot' + ? (event.type === 'batch' ? event.batch.items : event.page.items).filter( + (item) => item.body.kind === 'status' && item.body.tone === 'error' + ) + : [] + ) + expect(framedRows.length).toBeGreaterThan(0) + }) + + it('names the start failure on queued messages when the attach fails after acquiring (W4′a)', async () => { + await host.close(SESSION) + const id = await accept('hello') + // The attach's own success record is the post-acquisition step that fails. + const record = vi.spyOn(store, 'recordOperationOutcome') + record.mockImplementation(async (input) => { + if (input.operationId !== id && input.outcome.status === 'succeeded') { + record.mockRestore() + throw new Error('record store write failed') + } + return AgentSessionRecordStore.prototype.recordOperationOutcome.call(store, input) + }) + + // Read through the open any reader takes, so a conversation the failure dropped is reopened + // and its message read as that reopen settles it. + let settled: AgentJournalSubmission | undefined + await eventually(async () => { + settled = await reopened(id) + expect(settled?.dispatchState).not.toBe('pending') + }) + // The message names the start that failed, not a close or a restart it never met. + expect(settled).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining('record store write failed') + }) + expect(await errorRows()).toEqual([settled?.reason]) + }) +}) + +describe('an attach that fails after indexing its child', () => { + it('leaves no child behind, so the next send starts one and is delivered', async () => { + await host.close(SESSION) + const owned: boolean[] = [] + host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status') { + owned.push(event.session.hostExecutionOwned === true) + } + } + }) + const first = await accept('hello') + // The attach's own success record is the step after `onAttached` indexed the child. + const record = vi.spyOn(store, 'recordOperationOutcome') + record.mockImplementation(async (input) => { + if (input.operationId !== first && input.outcome.status === 'succeeded') { + record.mockRestore() + throw new Error('record store write failed') + } + return AgentSessionRecordStore.prototype.recordOperationOutcome.call(store, input) + }) + await eventually(async () => expect((await submission(first))?.dispatchState).toBe('rejected')) + // Nothing was indexed, so nothing had to be taken back: no list ever showed a child. + expect(host['sessions'].get(SESSION)?.child).toBeNull() + expect(owned).not.toContain(true) + const acquiresBefore = acquire.mock.calls.length + + const next = await accept('after the failure') + + await eventually(async () => expect((await submission(next))?.dispatchState).toBe('accepted')) + expect(acquire).toHaveBeenCalledTimes(acquiresBefore + 1) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([next]) + }) +}) + +describe('what an earlier host process left behind', () => { + it('rejects a message it accepted and never handed over, as not sent (W4′b)', async () => { + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('queued', 'q', true), fence }) + }) + + await host.revealSession(SESSION) + + expect(await submission('queued')).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_HOST_RESTARTED + }) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('leaves a legacy pending message and a handed-over one in doubt, never re-sent (W4′c)', async () => { + const providerHistoryWindow = vi.fn(async () => null) + adapterExtras = { providerHistoryWindow } + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('legacy', 'l'), fence }) + await journal.appendSubmission({ ...earlierSubmission('handed', 'h', true), fence }) + await journal.resolveDispatch({ clientMessageId: 'handed', state: 'pending', fence }) + }) + await host.flushAllStreamedEvents() + await startHost() + + await host.revealSession(SESSION) + + expect(await submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true }) + expect(await submission('handed')).toMatchObject({ dispatchState: 'unknown', recovered: true }) + // Deciding them from provider history waits for a won lease (W4′d). + expect(providerHistoryWindow).not.toHaveBeenCalled() + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('a child that exits before its message is handed over', () => { + it('rejects the message with the exit reason instead of starting another child (W24)', async () => { + // Each child the loop starts dies between its start step and its handover step. + const awaitStarted = vi.fn(async (sessionId: string) => { + await host.handleAdapterEvent({ + type: 'ended', + sessionId, + fence: store.getRecord(sessionId)!.lease.runtimeFence, + acquisitionGeneration: `generation-${acquire.mock.calls.length}`, + reason: 'codex app-server crashed', + cause: 'unexpected-exit' + }) + }) + adapterExtras = { awaitStarted } + await host.close(SESSION) + await startHost() + + const id = await accept('hello') + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect((await submission(id))?.reason).toContain('codex app-server crashed') + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('Stop withdraws what is queued', () => { + it('withdraws a crash leftover ahead of any delivery step (W17a)', async () => { + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) + }) + + // Stop's own open wakes the delivery loop, whose first step queues behind this Stop. + expect(await stop()).toMatchObject({ ok: true }) + + expect(await submission('leftover')).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + expect(acquire).toHaveBeenCalledTimes(1) + }) + + it('withdraws a message whose start holds the queue: nothing is handed over (W17b)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + const id = await accept('hello') + await eventually(async () => expect(acquire).toHaveBeenCalledTimes(2)) + const stopped = stop() + + starting.resolve() + expect(await stopped).toMatchObject({ ok: true }) + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect(await submission(id)).toMatchObject({ reason: DISPATCH_REJECTED_CANCELLED }) + expect((await submission(id))?.handedOverAt).toBeUndefined() + expect(dispatch).not.toHaveBeenCalled() + }) + + it('stops a child still proving its start, and the delivery loop ends with it (W17c)', async () => { + const ended = deferred() + const awaitStarted = vi.fn(() => ended.promise) + const closeSession = vi.fn(async () => { + ended.resolve() + return true + }) + adapterExtras = { awaitStarted, closeSession } + await host.close(SESSION) + await startHost() + acquire.mockImplementationOnce(async (input) => ({ + ...(await spawnChild(input)), + providerChildPhase: 'starting' as const + })) + const id = await accept('hello') + await eventually(async () => expect(awaitStarted).toHaveBeenCalled()) + + expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + + expect(await reopened(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + expect(closeSession).toHaveBeenCalled() + // A loop still waiting on that start would swallow this send; it starts a new child instead. + awaitStarted.mockImplementation(async () => undefined) + const next = await accept('after stop') + await eventually(async () => expect((await submission(next))?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) + }) +}) + +describe('an eviction between acceptance and handover', () => { + it('rejects the message as not sent, never leaves it in doubt (W24)', async () => { + const started = deferred() + adapterExtras = { awaitStarted: () => started.promise } + await host.close(SESSION) + await startHost() + const id = await accept('hello') + await eventually(async () => expect(acquire).toHaveBeenCalledTimes(2)) + // Between the delivery loop's start step and its handover step. + await host.close(SESSION) + started.resolve() + + expect(await reopened(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('rejects a queued message behind a handed-over one, which alone stays in doubt (W24)', async () => { + const second = deferred() + const awaitStarted = vi.fn(async (): Promise => undefined) + adapterExtras = { awaitStarted } + await host.close(SESSION) + await startHost() + dispatch.mockResolvedValueOnce({ state: 'admitted' }) + const handed = await accept('handed over') + await eventually(async () => expect((await submission(handed))?.handedOverAt).toBeDefined()) + awaitStarted.mockImplementation(() => second.promise) + const queued = await accept('still queued') + await eventually(async () => expect(awaitStarted).toHaveBeenCalledTimes(2)) + + await host.close(SESSION) + second.resolve() + + expect(await reopened(queued)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + expect(await submission(handed)).toMatchObject({ dispatchState: 'unknown' }) + expect(dispatch).toHaveBeenCalledTimes(1) + }) + + it('does not stop an idle child while a message is still queued for it (W24)', async () => { + idleMs = 0 + const started = deferred() + adapterExtras = { awaitStarted: () => started.promise } + await host.close(SESSION) + await startHost() + const id = await accept('hello') + await eventually(async () => expect(acquire).toHaveBeenCalledTimes(2)) + // The idle sweep ticks every few milliseconds meanwhile. + await new Promise((resolve) => setTimeout(resolve, 20)) + + started.resolve() + // Handed to the child it was queued for; the eviction may follow once nothing is owed. + await eventually(async () => expect(dispatch).toHaveBeenCalledTimes(1)) + expect(dispatch.mock.calls[0]?.[0].clientMessageId).toBe(id) + expect(acquire).toHaveBeenCalledTimes(2) + }) +}) + +describe('a compaction or rewind an earlier child left prepared', () => { + async function leftPrepared(prepare: (fence: number) => Promise): Promise { + await host.close(SESSION) + await prepare(store.getRecord(SESSION)!.lease.runtimeFence) + // A new process: nothing is open and no view attaches. + await host.flushAllStreamedEvents() + await startHost() + } + + it('settles an interrupted compaction at open, so a send is accepted and delivered (R16)', async () => { + await leftPrepared((fence) => + store.setConversationCommand(SESSION, fence, { + command: 'compact', + runtimeFence: fence, + operationId: 'compact-op', + callerKey: 'client-1', + phase: 'prepared', + state: 'unknown' + }) + ) + + const id = await accept('after the compaction') + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(store.getRecord(SESSION)?.conversationCommand).toMatchObject({ + phase: 'committed', + state: 'unknown' + }) + }) + + it('completes a rewind the provider already applied at open, so a send is accepted (R16)', async () => { + await leftPrepared((fence) => + persistRewindRecord(store, SESSION, fence, { + operationId: 'rewind-op', + callerKey: 'client-1', + itemId: 'orca:rewound', + providerItemId: `codex:${THREAD}:turn-1:0`, + expectedEpoch: 'epoch-before', + phase: 'provider-succeeded', + hydrationVerified: true, + retained: [] + }) + ) + + const id = await accept('after the rewind') + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(store.getRecord(SESSION)?.rewind).toMatchObject({ phase: 'completed' }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts index 3172a04597e6..288623893c5d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts @@ -15,6 +15,7 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach } from './structured-agent-session-attach-flow' import type { AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' @@ -154,6 +155,7 @@ describe('structured session acquisition options', () => { store: initialStore, adapter: withHistory('created'), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -194,6 +196,7 @@ describe('structured session acquisition options', () => { store, adapter: withHistory('resumed'), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-b', claimKeyId: 'key-1', @@ -228,6 +231,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -260,6 +264,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken, claimKeyId: 'key-1', @@ -291,6 +296,7 @@ describe('structured session acquisition options', () => { store, adapter: adapter({ origin: 'created' }), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -330,6 +336,7 @@ describe('structured session acquisition options', () => { } }), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-b', claimKeyId: 'key-1', @@ -369,6 +376,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -408,6 +416,7 @@ describe('structured session acquisition options', () => { store, adapter: failingAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -499,6 +508,7 @@ describe('structured session acquisition options', () => { store: target, adapter: failingAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!, failingAdapter), authority: { spawnToken: operationId === CREATE_OPERATION ? 'spawn-a' : 'spawn-b', claimKeyId: 'key-1', @@ -585,6 +595,7 @@ describe('the tab a create reserves', () => { store, adapter: adapter({ origin: 'created' }), journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts index 7fa1446d6c53..b209bbccb9e1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts @@ -61,8 +61,8 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi dispatch: StructuredAgentSessionAdapter['dispatch'] = (input) => this.owner(input.sessionId).dispatch(input) - rewindSupport: NonNullable = (sessionId) => - this.liveOwnerOrNull(sessionId)?.rewindSupport?.(sessionId) ?? { + rewindSupport: NonNullable = (sessionId, agent) => + this.capabilityOwner(sessionId, agent)?.rewindSupport?.(sessionId) ?? { supported: false, reason: 'unsupported' } @@ -94,11 +94,11 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi return change(input) } - supportsThreadGoal = (sessionId: string): boolean => - this.liveOwnerOrNull(sessionId)?.supportsThreadGoal?.(sessionId) ?? false + supportsThreadGoal = (sessionId: string, agent?: string): boolean => + this.capabilityOwner(sessionId, agent)?.supportsThreadGoal?.(sessionId) ?? false - recordsContextUsage = (sessionId: string): boolean => - this.liveOwnerOrNull(sessionId)?.recordsContextUsage?.(sessionId) ?? false + recordsContextUsage = (sessionId: string, agent?: string): boolean => + this.capabilityOwner(sessionId, agent)?.recordsContextUsage?.(sessionId) ?? false stopBackgroundTasks: NonNullable = ( input @@ -122,6 +122,8 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi awaitOptionWritable = (sessionId: string): Promise => this.liveOwnerOrNull(sessionId)?.awaitOptionWritable?.(sessionId) ?? Promise.resolve() + awaitStarted = (sessionId: string): Promise => + this.liveOwnerOrNull(sessionId)?.awaitStarted?.(sessionId) ?? Promise.resolve() readOptions = (input: { sessionId: string; fence: number }) => { const reader = this.owner(input.sessionId).readOptions @@ -214,6 +216,11 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi return adapter } + /** The live owner, or for a session at rest the provider it would start under. */ + private capabilityOwner(sessionId: string, agent?: string): StructuredAgentSessionAdapter | null { + return this.liveOwnerOrNull(sessionId) ?? (agent ? this.adapterForAgent(agent) : null) + } + private liveOwnerOrNull(sessionId: string): StructuredAgentSessionAdapter | null { const route = this.routes.get(sessionId) return route?.state === 'live' ? route.adapter : null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index c99b16d38a82..22e42be27221 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -208,7 +208,8 @@ export type StructuredAgentSessionAdapter = { /** Revalidate after preparation, immediately before writing to the provider. */ beforeDispatch?: () => Promise }): Promise - rewindSupport?(sessionId: string): AgentSessionRewindSupport + /** `agent` answers for a session with no child running, from the provider alone. */ + rewindSupport?(sessionId: string, agent?: string): AgentSessionRewindSupport recoverRewind?(input: { sessionId: string fence: number @@ -259,10 +260,10 @@ export type StructuredAgentSessionAdapter = { * start a new goal rather than rewrite that one's objective in place. */ replacesGoal: boolean }): Promise<{ ok: true } | { ok: false; rejected: string }> - /** Whether this live session can change its goal. */ - supportsThreadGoal?(sessionId: string): boolean - /** Whether this live session writes context facts to its turn rows. */ - recordsContextUsage?(sessionId: string): boolean + /** Whether this session can change its goal; `agent` answers one at rest. */ + supportsThreadGoal?(sessionId: string, agent?: string): boolean + /** Whether this session writes context facts to its turn rows; `agent` answers one at rest. */ + recordsContextUsage?(sessionId: string, agent?: string): boolean stopBackgroundTasks?(input: { sessionId: string fence: number @@ -288,6 +289,10 @@ export type StructuredAgentSessionAdapter = { ): Promise>> /** Resolves once a live session can take an option write, or after a bound; never rejects. */ awaitOptionWritable?(sessionId: string): Promise + /** Resolves once a session published before it proved its start has proven it, failed, or been + * closed; at once for any other. A start that did not land resolves with the chat's words for + * why. Never rejects. */ + awaitStarted?(sessionId: string): Promise readOptions?(input: { sessionId: string; fence: number }): Promise /** Option keys skipped after a provider rejected their persisted restore value. */ readOptionRestoreFailures?(sessionId: string): readonly string[] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts index 0248799980a7..4b8e1d87ba39 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts @@ -11,10 +11,12 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach, type AttachFlowInput } from './structured-agent-session-attach-flow' import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import * as legacyImport from '../agent-session-journal/journal-legacy-import' +import { StructuredAgentSessionHost } from './structured-agent-session-host' const NOW = 1_800_000_000_000 const SESSION = 'codex_adopting_session' @@ -121,6 +123,7 @@ async function attach( store, adapter: sessionAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -210,7 +213,7 @@ describe('adopting a provider conversation on create', () => { } ) - it('still releases acquisition and closes the provisional journal on an import write failure', async () => { + it('still releases acquisition on an import write failure, and leaves the conversation open', async () => { root = await mkdtemp(join(tmpdir(), 'orca-adopt-write-failure-')) const transcriptPath = join(root, 'rollout.jsonl') await writeCodexRollout(transcriptPath, 'valid source') @@ -222,7 +225,51 @@ describe('adopting a provider conversation on create', () => { await expect(attach(transcriptPath, sessionAdapter)).rejects.toThrow('disk write failed') expect(sessionAdapter.acquire).toHaveBeenCalledTimes(1) expect(sessionAdapter.releaseAcquisition).toHaveBeenCalledTimes(1) - expect(close).toHaveBeenCalledTimes(1) + // The journal is the conversation's, not the attach's: a failed import closes nothing. + expect(close).not.toHaveBeenCalled() + }) + + it('leaves the conversation writable when the import fails after acquiring', async () => { + root = await mkdtemp(join(tmpdir(), 'orca-adopt-host-failure-')) + const transcriptPath = join(root, 'rollout.jsonl') + await writeCodexRollout(transcriptPath, 'valid source') + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + const host = new StructuredAgentSessionHost({ + store, + adapter: adapter(), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => NOW + }) + vi.spyOn(AgentSessionJournal.prototype, 'replaceEpochItems').mockRejectedValueOnce( + new Error('disk write failed') + ) + const attached = await host + .attach({ callerKey: 'client-1' }, attachParams(transcriptPath)) + .catch(() => null) + expect(attached?.ok).not.toBe(true) + + const body = { kind: 'message' as const, role: 'user' as const, blocks: [] } + const sent = await host.send( + { callerKey: 'client-1' }, + { + envelope: { + sessionId: SESSION, + clientOperationId: `${NOW}-${'2'.padStart(32, '0')}`, + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } + ) + // The failed attach kept the conversation's own journal open, so the send is recorded. + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + await host.flushAllStreamedEvents() }) it('prepares a valid source once before acquisition and imports those exact items', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts index 459d21b1f3b7..7a85de6a473e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts @@ -1,7 +1,6 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionAttachParams, AttachedJournal } from './structured-agent-session-attach' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import type { JournalReplacementItem } from '../agent-session-journal/journal-epoch-replacement' import { importLegacyTranscriptIntoJournal, @@ -62,13 +61,8 @@ export async function importAdoptedTranscript( record: AgentSessionRecord, prepared: JournalReplacementItem[] | null ): Promise { - try { - await applyAdoptedTranscript(params, attached, record, prepared) - } catch (error) { - // Publication has not taken ownership of this provisional journal yet. - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error - } + // The journal is the conversation's, which outlives a failed import; nothing here closes it. + await applyAdoptedTranscript(params, attached, record, prepared) } async function applyAdoptedTranscript( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts similarity index 60% rename from src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume.ts rename to src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts index 9ea59e18487c..4a7ac433d206 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts @@ -1,11 +1,9 @@ // Giving a session its provider child back. // -// This is the replacement for the startup resume, and the difference is only in WHO asks: the same -// eligibility rule, run when a surface binds, when a send finds the owner gone, or when a child -// exits under an open surface — never when the app launches. It runs inside the session's +// Only work starts one: the delivery loop for a queued message, and the few operations that need +// the provider itself — never a view, and never the app launching. It runs inside the session's // serialize, with the attach it is given, so the eligibility it reads is the one the attach acts -// on. A write-capable hold must fail when acquisition is refused so the surface never mistakes a -// readable journal for a live provider child. +// on. import type { AgentSessionAttachResult, @@ -14,12 +12,10 @@ import type { } from '../../../shared/agent-session-wire' import { isAgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' -import { - attachStructuredAgentSessionUnderSerialize, - type StructuredAgentSessionAttachOptions -} from './structured-agent-session-attach-orchestration' +import { attachStructuredAgentSessionUnderSerialize } from './structured-agent-session-attach-orchestration' import { failedCreateRefusal } from './structured-agent-session-failed-create-refusal' import { adapterSupportsRecord } from './structured-agent-session-provider-support' +import { retryPendingStructuredAgentSessionSettlement } from './structured-agent-session-settlement-retry' import { structuredAgentSessionResumeOperationId, structuredAgentSessionResumeParams @@ -31,14 +27,55 @@ export type StructuredAgentSessionResumeOutcome = | { ok: true } | { ok: false; refusal: AgentSessionWireRefusal } -export async function resumeHeldStructuredAgentSession(input: { +/** The attach's caller key: the ledger row a start settles is Orca's own. */ +const AGENT_START_CALLER_KEY = 'trusted-local:agent-start' + +/** + * Gives the session a provider child if it has none, for a caller inside its serialize — which is + * what makes "if it has none" exact: two askers run this in turn, and the second finds the first + * one's child. A failed start leaves the next asker to make its own. + */ +export async function ensureStructuredAgentSessionAgent( + context: StructuredAgentSessionAttachContext, + sessionId: string, + startedFor?: string +): Promise { + if (context.sessions.get(sessionId)?.child) { + return { ok: true } + } + const started = await startStructuredAgentSessionAgent(context, sessionId, startedFor) + if (!started.ok || context.sessions.get(sessionId)?.child) { + return started + } + return { + ok: false, + refusal: { + code: 'agent_session_ownership_unknown', + message: 'The session attached without a provider child to write to.' + } + } +} + +/** The same, for an operation's admission: a start that throws is that operation's refusal. */ +export function ensureStructuredAgentSessionAgentForOperation( + context: StructuredAgentSessionAttachContext, sessionId: string - context: StructuredAgentSessionAttachContext - /** Who is asking; the attach keys the ledger row it settles by it. */ - callerKey: string - attachOptions?: StructuredAgentSessionAttachOptions -}): Promise { - const { sessionId, context, callerKey } = input +): Promise { + return ensureStructuredAgentSessionAgent(context, sessionId).catch((error: unknown) => ({ + ok: false, + refusal: { + code: 'agent_session_owner_restart_failed', + message: error instanceof Error ? error.message : String(error) + } + })) +} + +async function startStructuredAgentSessionAgent( + context: StructuredAgentSessionAttachContext, + sessionId: string, + startedFor: string | undefined +): Promise { + const callerKey = AGENT_START_CALLER_KEY // The record is read only once this host has adjudicated it and exited any recovery stage a // failed attempt latched — a lease left in `manual-recovery` by an unproven exit is one the // resolver hands back, and the eligibility below must see it that way. @@ -47,6 +84,15 @@ export async function resumeHeldStructuredAgentSession(input: { return { ok: false, refusal: unreconciled } } await context.runtimeState.resolveRecovery(sessionId) + // An exit whose journal settlement failed latches the lease until a retry lands, and the recovery + // resolver never clears that latch. The start is that retry, so the send that needs the agent + // settles it. + await retryPendingStructuredAgentSessionSettlement({ + deps: context.deps, + sessionId, + openJournal: async () => (await context.openConversation(sessionId))?.journal ?? null, + now: () => context.now() + }) const record = context.deps.store.getRecord(sessionId) if (!record) { return refuse('agent_session_identity_required', 'No structured session exists by that id.') @@ -80,7 +126,7 @@ export async function resumeHeldStructuredAgentSession(input: { context, callerKey, params, - input.attachOptions + startedFor === undefined ? {} : { startedFor } ) } catch (error) { // The attach settles an acquisition that failed — the ledger row, the released lease — before diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts index c8c908aae70c..9f38c85755f5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts @@ -150,11 +150,15 @@ describe('an open chat receives every row its journal commits', () => { await exitBeforeProof() - expect(pane.received().statuses).toEqual([ - expect.stringMatching(/stopped before it finished starting: .*not signed in/) - ]) - expect(pane.received().submissions).toContainEqual( - expect.objectContaining({ clientMessageId: held, dispatchState: 'rejected' }) + // The exit ends the child; the delivery loop, which reads why, rejects what it had queued. + await vi.waitFor(() => + expect(pane.received().submissions).toContainEqual( + expect.objectContaining({ clientMessageId: held, dispatchState: 'rejected' }) + ) + ) + // One row, however many of its writers reported the start. + expect(new Set(pane.received().statuses)).toEqual( + new Set([expect.stringMatching(/stopped before it finished starting: .*not signed in/)]) ) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts index 4b9b46716cb2..f792ac588d6a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts @@ -38,8 +38,7 @@ export type StructuredAgentSessionAttachContext = { reconcileLeases: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise now: () => number - /** Paired with `sessions.delete` by `forgetStructuredAgentSession`; a failed attach that only - * deleted would leave the store's row behind. */ - forgetStatus: (sessionId: string) => void - publishStatus?: (sessionId: string) => void + publishStatus: (sessionId: string) => void + /** The conversation's one open journal, opened when closed; see `conversation-open`. */ + openConversation: (sessionId: string) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts index 3a77aa2d6cca..6af8eb369d52 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts @@ -24,8 +24,7 @@ export async function settlePostAcquisitionAttachFailure( ? 'root-exit-observed' : 'exit-proven' } - // A failed close must not prevent durable failure settlement. - await Promise.resolve(input.onAttachFailed?.()).catch(() => undefined) + input.onAcquisitionReleased?.(cause, { rootGone: exitProof !== 'unproven' }) try { await input.store.settleFailedPostAcquisitionAttachment({ sessionId: record.sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts index cf37a9a6fe2d..2969cb62f09b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts @@ -41,6 +41,7 @@ import { type AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' export type AttachFlowInput = { store: AgentSessionRecordStore @@ -66,8 +67,11 @@ export type AttachFlowInput = { onAcquiring?: () => Promise | void /** Settles writes already captured by the superseded journal before opening another. */ beforeJournalOpen?: () => Promise | void - /** Closes and removes partial publication after journal attachment fails. */ - onAttachFailed?: () => Promise + /** The conversation's own open journal, which the attach adopts: it never opens one itself. */ + openConversation: (record: AgentSessionRecord) => Promise + /** A failure after acquisition released the session's acquisition; `cause` is that failure and + * `rootGone` whether the release saw the provider root go. */ + onAcquisitionReleased?: (cause: unknown, verdict: { rootGone: boolean }) => void } export async function performAttach( @@ -208,6 +212,7 @@ export async function performAttach( params, journalRoot: input.journalRoot, adapter: input.adapter, + openConversation: input.openConversation, providerHistoryWindow }) await importAdoptedTranscript(params, attached, record, preparedTranscript.items) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index f39a183b0f09..e35a2db0baa7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -24,9 +24,16 @@ import { refuseAgentSessionMutation } from './structured-agent-session-mutation- import { retryPendingStructuredAgentSessionSettlement } from './structured-agent-session-settlement-retry' import { settleStaleSessionStateOnAcquire } from './structured-agent-session-stale-turn-verdict' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' -import { forgetStructuredAgentSession } from './structured-agent-session-host-lifetime' +import type { + StructuredAgentSessionProviderChild, + StructuredAgentSessionStopVerdict +} from './structured-agent-session-host-types' +import { + endProviderChild, + indexProviderChild, + structuredAgentSessionConversationFence +} from './structured-agent-session-provider-child' import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { addAgentSessionCreatePhaseAttributes, @@ -36,17 +43,17 @@ import { } from '../../observability/agent-session-instrumentation' export type StructuredAgentSessionAttachOptions = { - /** Provider-exit recovery: refuses once the ticket the restart was issued for is stale. */ - admitRecoveryTicket?: () => boolean recordPhase?: AgentSessionCreatePhaseRecorder + /** The queued message a start is for; see `StructuredAgentSessionProviderChild.startedFor`. */ + startedFor?: string } /** * The attach itself, for a caller already inside the session's serialize. * - * That is every caller that has to know what the session looks like RIGHT NOW: a hold, a send - * making sure it has an owner, provider-exit recovery. They run their - * check and this attach in one serialized step, so "the session has no child" is still true when + * That is every caller that has to know what the session looks like RIGHT NOW: the delivery loop, + * and an operation that needs the provider. They run their check and this attach in one serialized + * step, so "the session has no child" is still true when * the attach starts. `attachStructuredAgentSession` is this under `serialize`, for a client. */ export function attachStructuredAgentSessionUnderSerialize( @@ -95,12 +102,10 @@ async function runAttach( ): Promise> { const sessionId = params.envelope.sessionId const recordPhase = options.recordPhase - if (options.admitRecoveryTicket && !options.admitRecoveryTicket()) { - return refuseAgentSessionMutation({ - code: 'agent_session_checkpoint_stale', - message: 'The provider-exit recovery ticket is no longer current.' - }) - } + // Readers of a conversation already open are re-baselined when this attach moves its fence. + const fenceBefore = context.sessions.has(sessionId) + ? structuredAgentSessionConversationFence(context.deps.store, sessionId) + : null const unreconciled = await withAgentSessionCreatePhase('reconcile_leases', recordPhase, () => context.reconcileLeases(sessionId) ) @@ -115,9 +120,8 @@ async function runAttach( const settled = await withAgentSessionCreatePhase('settlement_retry', recordPhase, () => retryPendingStructuredAgentSessionSettlement({ deps: context.deps, - sessions: context.sessions, sessionId, - params, + openJournal: async () => (await context.openConversation(sessionId))?.journal ?? null, now: () => context.now() }) ) @@ -131,14 +135,15 @@ async function runAttach( context.runtimeState.probeOwner(sessionId) ) // A child this attach spawns writes through a sink this attempt owns. Only a successful - // attach makes it the session's; any other exit closes it with whatever the child queued. + // attach makes the child and its sink the session's; any other exit closes the sink with + // whatever the child queued, and leaves the conversation's child as it was. const attemptSink = context.runtimeState.mintEventSink(sessionId) - let attemptSinkAdopted = false - const attached = stampFailedCreateOwnerVerdict( - context.deps.store, - callerKey, - params.envelope, - await performAttach({ + const attempt: { candidate: AttachCandidate | null; committed: boolean } = { + candidate: null, + committed: false + } + try { + const attached = await performAttach({ store: context.deps.store, adapter: context.deps.adapter, journalRoot: context.deps.journalRoot, @@ -162,65 +167,46 @@ async function runAttach( params, now: () => context.now(), recordPhase, - // Site 9: this closes the PRIOR map entry it drops, never the provisional - // journal — it has no reference to that one. `onAttached` owns that. - onAttachFailed: async () => { - await forgetStructuredAgentSession(context, sessionId) - context.runtimeState.currentEventSink(sessionId)?.close() - context.runtimeState.discardEventSink(sessionId) + openConversation: async (record) => { + const conversation = await context.openConversation(record.sessionId) + if (!conversation) { + throw new Error('agent_session_identity_required') + } + return conversation.journal }, + // The cleanup released the acquisition, which for a re-attach is the live child itself. + onAcquisitionReleased: (cause, verdict) => + endReleasedChild(context, sessionId, cause, verdict), onAttached: async (attached, acquisitionGeneration, acquiredOwner, providerChildPhase) => { - const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? 0 - const previous = context.sessions.get(sessionId) - const previousFence = previous?.fence + const fence = structuredAgentSessionConversationFence(context.deps.store, sessionId) + const current = context.sessions.get(sessionId)?.child ?? null + const startedFor = acquiredOwner ? options.startedFor : current?.startedFor // A re-attach to a live child keeps the sink that child already writes through. const eventSink = acquiredOwner ? attemptSink : (context.runtimeState.currentEventSink(sessionId) ?? attemptSink) - // Site 8: the provisional journal has no owner until the map takes it, - // and the barrier below throws by design. - try { - if (acquiredOwner) { - // Before the drain: the buffered events are the new child's, never a stale row's. - await settleStaleSessionStateOnAcquire({ - journal: attached.journal, - sessionId, - fence, - acquisitionGeneration - }) - } - await bindAndDrain(eventSink, attached.journal, fence, (activity) => - context.subscribers.publish(sessionId, attached.journal, activity) - ) - } catch (error) { - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error + if (acquiredOwner) { + // Before the drain: the buffered events are the new child's, never a stale row's. + await settleStaleSessionStateOnAcquire({ + journal: attached.journal, + sessionId, + fence, + acquisitionGeneration + }) } - // Site 10: a `set` over a live entry would orphan its handle — and a - // close that REJECTED did not release it. The replacement is therefore - // ABORTED rather than completed over a handle nothing can reach again: - // `previous` stays indexed, so teardown still owns it and can retry. - if (previous && previous.journal !== attached.journal) { - try { - await previous.journal.close() - } catch (error) { - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error + await bindAndDrain(eventSink, attached.journal, fence, (activity) => + context.subscribers.publish(sessionId, attached.journal, activity) + ) + attempt.candidate = { + sink: eventSink, + child: { + generation: acquisitionGeneration ?? current?.generation ?? null, + fence, + // A re-attach to a live child keeps what that child already proved, and its cause. + phase: acquiredOwner ? providerChildPhase : (current?.phase ?? 'ready'), + ...(startedFor === undefined ? {} : { startedFor }) } } - context.runtimeState.adoptEventSink(sessionId, eventSink) - attemptSinkAdopted = eventSink === attemptSink - context.sessions.set(sessionId, { - journal: attached.journal, - params, - fence, - hasProviderChild: true, - // A re-attach to a live child keeps what that child already proved. - providerChildPhase: acquiredOwner - ? providerChildPhase - : (previous?.providerChildPhase ?? 'ready'), - acquisitionGeneration: acquisitionGeneration ?? previous?.acquisitionGeneration ?? null - }) await recoverStructuredRewind( context.deps.store, sessionId, @@ -230,21 +216,59 @@ async function runAttach( context.now ) await recoverInterruptedCompaction(context.deps.store, sessionId, attached.journal, fence) - if (attached.recovery) { - context.subscribers.reset(sessionId, attached.journal, attached.recovery.reset, fence) - } else if (previousFence !== undefined && previousFence !== fence) { + if (fenceBefore !== null && fence !== fenceBefore) { context.subscribers.snapshot(sessionId, attached.journal, fence) } else { context.subscribers.publish(sessionId, attached.journal) } } - }).finally(() => { - if (!attemptSinkAdopted) { - attemptSink.close() - } }) - ) - return attached + const { candidate } = attempt + const conversation = context.sessions.get(sessionId) + if (attached.ok && candidate && conversation) { + context.runtimeState.adoptEventSink(sessionId, candidate.sink) + attempt.committed = candidate.sink === attemptSink + indexProviderChild(conversation, candidate.child) + context.publishStatus?.(sessionId) + } + return stampFailedCreateOwnerVerdict(context.deps.store, callerKey, params.envelope, attached) + } finally { + if (!attempt.committed) { + attemptSink.close() + } + } +} + +type AttachCandidate = { + child: StructuredAgentSessionProviderChild + sink: DeferredStructuredAgentSessionEventSink +} + +function endReleasedChild( + context: StructuredAgentSessionAttachContext, + sessionId: string, + cause: unknown, + verdict: StructuredAgentSessionStopVerdict +): void { + const session = context.sessions.get(sessionId) + const child = session?.child + if ( + !session || + !child || + !endProviderChild(session, { + generation: child.generation, + fence: child.fence, + cause: 'attach-failed', + reason: cause instanceof Error ? cause.message : String(cause), + duringStartup: child.phase === 'starting', + ...verdict + }) + ) { + return + } + context.runtimeState.currentEventSink(sessionId)?.close() + context.runtimeState.discardEventSink(sessionId) + context.publishStatus?.(sessionId) } /** Binds the sink to the journal and waits for the barrier the host publishes diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts index 97f429933d87..4e15b23d4d3f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts @@ -13,6 +13,7 @@ import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { attachJournal, journalIdentityFor, @@ -85,6 +86,7 @@ async function attach(adapter: StructuredAgentSessionAdapter) { record: RECORD, params: PARAMS, journalRoot: root, + openConversation: openTestAttachConversation(root), adapter }) journals.track(attached.journal) @@ -147,4 +149,37 @@ describe('attachJournal restart reconciliation', () => { expect(attached.unconfirmedClientMessageIds).toEqual(['cm_1']) expect(attached.journal.submissions()[0]?.dispatchState).toBe('unknown') }) + + it('leaves a message the open conversation still has queued alone (W4′e)', async () => { + const journal = await journals.open({ + identity: IDENTITY, + journalDir: journalDirectoryFor(root, { + workspaceId: IDENTITY.workspaceId, + sessionId: IDENTITY.sessionId + }) + }) + await journal.appendSubmission({ + clientMessageId: 'queued', + payloadFingerprint: digestPayload('still queued'), + body: userMessage('still queued'), + fence: RECORD.lease.runtimeFence, + handoverRecorded: true + }) + // History that holds nothing: absence would prove a handed-over message undelivered. + const { adapter, dispatch } = adapterWith(async () => window()) + + const attached = await attachJournal({ + record: RECORD, + params: PARAMS, + journalRoot: root, + + adapter, + openConversation: async () => journal + }) + + expect(attached.journal).toBe(journal) + expect(journal.submissions()[0]).toMatchObject({ dispatchState: 'pending' }) + expect(journal.submissions()[0]?.handedOverAt).toBeUndefined() + expect(dispatch).not.toHaveBeenCalled() + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts new file mode 100644 index 000000000000..4b8037077aa8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts @@ -0,0 +1,15 @@ +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { openStructuredAgentSessionConversationJournal } from './structured-agent-session-conversation-open' + +/** For a test that attaches without a host: the conversation's journal, through the one open a + * host would take, so the attach under test adopts it the way it adopts the host's. */ +export function openTestAttachConversation( + journalRoot: string, + adapter: Pick = {} +): (record: AgentSessionRecord) => Promise { + return async (record) => + (await openStructuredAgentSessionConversationJournal({ journalRoot, adapter }, record)).session + .journal +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts index de5fa297daf8..4cf6505d2768 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts @@ -35,15 +35,9 @@ import { } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { agentSessionProviderHandleChainHead } from '../../../shared/agent-session-provider-handle' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' -import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { reconcileJournalSubmissionsAgainstHistory } from '../agent-session-journal/journal-restart-reconciliation' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' -import { - openAgentSessionJournalWithRecovery, - type AgentSessionJournalRecovery -} from './agent-session-journal-recovery' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { structuredAgentSessionRefusalMessage } from './structured-agent-session-refusal-message' @@ -165,16 +159,13 @@ export function journalIdentityFor( export type AttachedJournal = { journal: AgentSessionJournal - recovery: AgentSessionJournalRecovery | null - /** Submissions still `unknown` after this open: the crash boundary settled - * them there and provider history could not decide them either. */ + /** Submissions the crash boundary left `unknown` that provider history could not decide. */ unconfirmedClientMessageIds: string[] } /** - * Open the session's journal, recovering it when the stored one is unusable, - * settle every submission left in flight by a previous process, then let - * provider history decide the ones it can prove. + * The conversation's journal — opened, and its crash boundary settled, by the conversation's own + * open — with provider history deciding the submissions that boundary could only doubt. * * Why the reconciliation belongs HERE and nowhere else: this runs after the * record store handed this host the lease and before `onAttached` starts a @@ -182,54 +173,44 @@ export type AttachedJournal = { * is read, and the window stays valid until the resume consumes it. Every other * settlement site — a proven child exit — runs while the host * may still start another child, and a read there could be overtaken before it - * is acted on. Orca still never re-sends: this decides state only. + * is acted on. Orca still never re-sends: this decides state only. A queued + * submission is left alone: it was never handed over, so history cannot hold it. */ export async function attachJournal(input: { record: AgentSessionRecord params: AgentSessionAttachParams journalRoot: string adapter: StructuredAgentSessionAdapter + /** The host's open conversation, whose journal the attach adopts. */ + openConversation: (record: AgentSessionRecord) => Promise /** Provider history sampled before a new child is acquired. `null` means the * adapter had no usable history; omit to read lazily for direct callers. */ providerHistoryWindow?: ProviderHistoryWindow | null }): Promise { const identity = journalIdentityFor(input.record, input.params) const fence = input.record.lease.runtimeFence - const historyFilePath = input.adapter.historyFilePath - ? await input.adapter.historyFilePath({ identity }) - : null - const opened = await openAgentSessionJournalWithRecovery({ + const journal = await input.openConversation(input.record) + const settled = await reconcileAgainstProviderHistory({ + adapter: input.adapter, identity, - journalDir: journalDirectoryFor(input.journalRoot, { - workspaceId: identity.workspaceId, - sessionId: identity.sessionId - }), + journal, fence, - historyFilePath + accountHome: input.record.accountHome, + ...(Object.hasOwn(input, 'providerHistoryWindow') + ? { history: input.providerHistoryWindow } + : {}) }) - try { - // That await is a WRITE. A failure in it leaves the journal with no caller - // holding a reference to close it. - const unconfirmed = await opened.journal.markPendingSubmissionsUnknown(fence) - const settled = await reconcileAgainstProviderHistory({ - adapter: input.adapter, - identity, - journal: opened.journal, - fence, - accountHome: input.record.accountHome, - ...(Object.hasOwn(input, 'providerHistoryWindow') - ? { history: input.providerHistoryWindow } - : {}) - }) - return { - ...opened, - unconfirmedClientMessageIds: unconfirmed.filter((id) => !settled.includes(id)) - } - } catch (error) { - // A rejected close leaves the handle open, so the journal is retained for a - // later retry rather than dropped along with the only reference to it. - await agentSessionJournalCloseRetries.closeOrRetain(opened.journal) - throw error + return { + journal, + unconfirmedClientMessageIds: journal + .submissions() + .filter( + (entry) => + entry.dispatchState === 'unknown' && + entry.recovered === true && + !settled.includes(entry.clientMessageId) + ) + .map((entry) => entry.clientMessageId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts index 5d922d723503..a88949a296f5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts @@ -12,21 +12,25 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' export class StructuredAgentSessionBackgroundTaskChannel { constructor( private readonly deps: StructuredAgentSessionHostDeps, private readonly sessions: Map, private readonly subscribers: AgentSessionSubscribers, - private readonly requireSession: (sessionId: string) => StructuredAgentSessionHostSession, + /** The host's accessor: opens a conversation at rest, and never starts an agent. */ + private readonly conversation: ( + sessionId: string + ) => Promise, /** Task edges change the status summary too; the feed's equality check * keeps a no-op re-projection from reaching subscribers. */ private readonly onPublished: (sessionId: string) => void ) {} - history(request: AgentSessionHistoryRequest): AgentSessionHistoryResult { + async history(request: AgentSessionHistoryRequest): Promise { const result = readStructuredAgentSessionHistoryResult({ - journal: this.requireSession(request.sessionId).journal, + journal: (await this.conversation(request.sessionId)).journal, record: this.deps.store.getRecord(request.sessionId), request }) @@ -42,13 +46,14 @@ export class StructuredAgentSessionBackgroundTaskChannel { } } - subscribe(input: AgentSessionSubscribeInput): () => void { - const session = this.requireSession(input.sessionId) + /** Resolves once the conversation is open and the subscriber holds its opening frame. */ + async subscribe(input: AgentSessionSubscribeInput): Promise<() => void> { + const session = await this.conversation(input.sessionId) const backgroundTasks = this.state(input.sessionId) return this.subscribers.open({ ...input, journal: session.journal, - fence: this.deps.store.getRecord(input.sessionId)?.lease.runtimeFence ?? 0, + fence: structuredAgentSessionConversationFence(this.deps.store, input.sessionId), ...(backgroundTasks !== undefined ? { backgroundTasks } : {}) }) } @@ -57,7 +62,11 @@ export class StructuredAgentSessionBackgroundTaskChannel { const session = this.sessions.get(sessionId) const state = publishedState !== undefined ? publishedState : this.state(sessionId) if (session && state !== undefined) { - this.subscribers.backgroundTasks(sessionId, state, session.fence) + this.subscribers.backgroundTasks( + sessionId, + state, + structuredAgentSessionConversationFence(this.deps.store, sessionId) + ) this.onPublished(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index a73e46b898e7..4a8f0a1440f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -11,7 +11,7 @@ import { import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { evictHeldStructuredAgentSession } from './structured-agent-session-host-lifetime' +import { stopStructuredAgentSessionAgentUnderSerialize } from './structured-agent-session-host-lifetime' import { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' @@ -24,7 +24,7 @@ afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) }) -describe('Claude root-exit eviction', () => { +describe('Claude root-exit stop', () => { it('releases a captured live claim after the provider root exits', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-claude-root-exit-')) roots.push(root) @@ -78,6 +78,7 @@ describe('Claude root-exit eviction', () => { journalDir: join(root, 'journal') }) const close = vi.spyOn(journal, 'close') + const publishStatus = vi.fn() const params: AgentSessionAttachParams = { envelope: { sessionId: 'session-1', @@ -103,10 +104,11 @@ describe('Claude root-exit eviction', () => { { journal, params, - fence, - hasProviderChild: true, - providerChildPhase: 'ready', - acquisitionGeneration: acquisition.acquisitionGeneration ?? null + child: { + generation: acquisition.acquisitionGeneration ?? null, + fence, + phase: 'ready' + } } ] ]) @@ -115,13 +117,13 @@ describe('Claude root-exit eviction', () => { claude.connections[0]!.handlers.onExit?.(new Error('provider exited')) await expect( - evictHeldStructuredAgentSession( + stopStructuredAgentSessionAgentUnderSerialize( { deps, runtimeState, sessions, now: () => NOW + 30 * 60_000, - forgetStatus: vi.fn() + publishStatus }, 'session-1' ) @@ -132,8 +134,10 @@ describe('Claude root-exit eviction', () => { ownerProcess: null, deathEvidence: { kind: 'exit-observed' } }) - expect(sessions.size).toBe(0) - expect(close).toHaveBeenCalledOnce() + // The agent went to rest; the conversation stays open and listed. + expect(sessions.get('session-1')?.child).toBeNull() + expect(close).not.toHaveBeenCalled() + expect(publishStatus).toHaveBeenCalledWith('session-1') // The adapter agrees the session is over: nothing is left to refuse the next start. await expect(adapter.closeSession('session-1')).resolves.toBe(true) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index aa3f298a0778..2915653bd375 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -1,4 +1,5 @@ import type { AgentChildWorkEvidence } from '../../../shared/agent-status-child-work-evidence' +import type { StructuredAgentSessionSavedStatus } from '../../../shared/structured-agent-session-saved-status' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { AgentSessionSubscribers } from './structured-agent-session-subscribers' import type { @@ -28,9 +29,15 @@ export class StructuredAgentSessionClientDelivery { private readonly sessions: Map, now: () => number, deps: () => StructuredAgentSessionHostDeps, - private readonly onJournalActivity?: (sessionId: string) => void + private readonly onJournalActivity?: (sessionId: string) => void, + onAgentStarted?: (sessionId: string) => void ) { - this.statusFeed = createStructuredAgentSessionHostStatusFeed({ sessions, now, deps }) + this.statusFeed = createStructuredAgentSessionHostStatusFeed({ + sessions, + now, + deps, + ...(onAgentStarted ? { onAgentStarted } : {}) + }) this.turnCompletionFeed = new StructuredAgentSessionTurnCompletionFeed({ sessions, now }) this.sendSettlement = new StructuredAgentSessionSendSettlement((sessionId) => this.requireJournal(sessionId) @@ -58,6 +65,10 @@ export class StructuredAgentSessionClientDelivery { publishRestored = (sessionId: string): void => this.statusFeed.publish(sessionId, undefined, { replay: true }) + seedStatus = (sessionId: string, saved: StructuredAgentSessionSavedStatus): void => { + this.statusFeed.seedRestored(sessionId, saved) + } + subscribeStatus = (subscriber: StructuredAgentSessionStatusSubscriber): (() => void) => this.statusFeed.subscribe(subscriber) forgetStatus = (sessionId: string): void => this.statusFeed.forget(sessionId) @@ -66,10 +77,16 @@ export class StructuredAgentSessionClientDelivery { subscriber: StructuredAgentSessionTurnCompletionSubscriber ): (() => void) => this.turnCompletionFeed.subscribe(subscriber) - closeSession(sessionId: string): void { + /** The conversation's handle closed. Its status row stays in every session list; the + * agent-status store keeps it too while the chat still has a tab to show it in. */ + closeSession(sessionId: string, options: { listed: boolean }): void { this.sendSettlement.closeSession(sessionId) - this.statusFeed.close(sessionId) - // The next attach re-baselines rather than announcing the turn it was already holding. + if (options.listed) { + this.statusFeed.revokeLive(sessionId) + } else { + this.statusFeed.close(sessionId) + } + // The next open re-baselines rather than announcing the turn it was already holding. this.turnCompletionFeed.forget(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts index b9e8f8e506d7..f5410de74f62 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts @@ -78,10 +78,7 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe return { journal, params: {} as StructuredAgentSessionHostSession['params'], - fence: 1, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: null } } @@ -138,7 +135,7 @@ function attachContext( reconcileLeases: async () => null, serialize: (_sessionId: string, task: () => Promise) => task(), now: () => 1, - forgetStatus: () => undefined + publishStatus: () => undefined } as unknown as StructuredAgentSessionAttachContext } @@ -178,40 +175,34 @@ describe('the registry', () => { }) describe('the attach orchestration', () => { - it('ABORTS the map replacement when the previous journal will not close', async () => { - const previousDir = join(root, 'previous') - const provisionalDir = join(root, 'provisional') - const previous = flakyClose( - await journals.open({ identity: IDENTITY, journalDir: previousDir }), - 1 - ) - const provisional = await journals.open({ - identity: IDENTITY, - journalDir: provisionalDir - }) - attachFlow.journal = provisional - const sessions = new Map([[SESSION, hostSession(previous)]]) + // The attach adopts the conversation's one open journal; it never opens a second handle, so + // there is no replacement to abort and no provisional journal to close. + it('keeps the journal it adopted indexed and open when an attach succeeds', async () => { + const directory = join(root, 'adopted') + const journal = await journals.open({ identity: IDENTITY, journalDir: directory }) + attachFlow.journal = journal + const sessions = new Map([[SESSION, hostSession(journal)]]) - await expect( - attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams) - ).rejects.toThrow('close rejected') + await attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams) - // The live entry is UNTOUCHED: overwriting it would have left its handle - // open with nothing able to reach it again. - expect(sessions.get(SESSION)?.journal).toBe(previous) - // And the provisional journal is owned by the registry, not orphaned. + expect(sessions.get(SESSION)?.journal).toBe(journal) + await expect( + journal.appendItem( + { provider: 'orca', clientMessageId: 'after-attach' }, + { + kind: 'status', + text: 'still writable' + } + ) + ).resolves.toBeDefined() expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([]) - await expectNothingHoldsTheDirectory(provisionalDir) }) - it('retains the provisional journal when its own close rejects on the barrier path', async () => { - const provisionalDir = join(root, 'provisional-barrier') - const provisional = flakyClose( - await journals.open({ identity: IDENTITY, journalDir: provisionalDir }), - 1 - ) - attachFlow.journal = provisional - const sessions = new Map() + it('leaves the conversation indexed and open when the sink barrier fails', async () => { + const directory = join(root, 'adopted-barrier') + const journal = await journals.open({ identity: IDENTITY, journalDir: directory }) + attachFlow.journal = journal + const sessions = new Map([[SESSION, hostSession(journal)]]) const context = attachContext(sessions) const failing = { sink: {}, @@ -229,9 +220,19 @@ describe('the attach orchestration', () => { 'sink barrier failed' ) - expect(sessions.size).toBe(0) - // Retained rather than dropped, so teardown can still release the handle. - expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([provisionalDir]) + // A failed attach does not end the conversation: its queued messages and the failure row + // are written into this same journal. + expect(sessions.get(SESSION)?.journal).toBe(journal) + await expect( + journal.appendItem( + { provider: 'orca', clientMessageId: 'after-failure' }, + { + kind: 'status', + text: 'still writable' + } + ) + ).resolves.toBeDefined() + expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([]) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts new file mode 100644 index 000000000000..74f916d57c1c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts @@ -0,0 +1,327 @@ +// Stopping an agent and closing its conversation's handle are two things, and a reader, a session +// list and a send each see only the one that happened. Ticks are driven by hand. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { DISPATCH_REJECTED_HOST_RESTARTED } from '../../../shared/structured-agent-session-dispatch-rejection' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { readStructuredSessionGateFacts } from '../../runtime/orchestration/structured-mailbox-pointer-host' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { setStructuredAgentSessionHost } from './structured-agent-session-registry' +import { + collectSubscriber, + createRestTestRig, + foundRestTestChat, + IDLE_MS, + readerSaw, + REST_TEST_CALLER as CALLER, + REST_TEST_SESSION as SESSION, + restTestSend, + sweepOnce, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { StructuredAgentSessionIdleSweep } from './structured-agent-session-idle-sweep' +import { hostTestAttachParams } from './structured-agent-session-host-test-data' + +let rig: RestTestRig + +const COLD_START = { timeout: 10_000 } + +beforeEach(async () => { + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) + setStructuredAgentSessionHost(rig.host) +}) + +afterEach(async () => { + setStructuredAgentSessionHost(null) + await rig.dispose() +}) + +function fence(): number { + return rig.store.getRecord(SESSION)?.lease.runtimeFence ?? 1 +} + +function openSession(): StructuredAgentSessionHostSession | undefined { + return rig.host.collaboratorsForTests().sessions.get(SESSION) +} + +function statusRows(): AgentSessionStatusSummary[] { + return rig.statusEvents + .flatMap((event) => (event.type === 'status' ? [event.session] : [])) + .filter((summary) => summary.sessionId === SESSION) +} + +describe('a stop that fails', () => { + it('is retried on the next tick (P2-11 a)', async () => { + await foundRestTestChat(rig) + rig.adapter.closeSession.mockResolvedValueOnce(false) + rig.clock.now += IDLE_MS + 1 + + await sweepOnce(rig.host) + expect(openSession()?.owesProviderChildWindDown).toMatchObject({ + generation: expect.any(String) + }) + await sweepOnce(rig.host) + + expect(rig.adapter.closeSession).toHaveBeenCalledTimes(2) + expect(rig.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + expect(statusRows().at(-1)?.hostExecutionOwned).toBeUndefined() + }) + + it('finishes its wind-down on the next tick, whatever its own writes did to the clock (P2-11 b)', async () => { + await foundRestTestChat(rig) + rig.clock.now += IDLE_MS + 1 + // The child is proven gone and its work settled, then handing the lease back fails. + const transition = vi + .spyOn(rig.store, 'transitionHandoff') + .mockRejectedValueOnce(new Error('store unavailable')) + + await sweepOnce(rig.host) + expect(transition).toHaveBeenCalledOnce() + expect(rig.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + // Five minutes later, not thirty. + rig.clock.now += 5 * 60_000 + await sweepOnce(rig.host) + + expect(rig.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + expect(rig.adapter.closeSession).toHaveBeenCalledOnce() + }) +}) + +describe('closing the handle', () => { + it('keeps a tabbed chat listed and its reader open, and a send reopens it for that reader (P2-14)', async () => { + await foundRestTestChat(rig) + const reader = collectSubscriber() + await rig.host.subscribe({ id: 'reader', sessionId: SESSION, emit: reader.emit }) + rig.clock.now += IDLE_MS + 1 + + await sweepOnce(rig.host) + expect(rig.host.hasSession(SESSION)).toBe(false) + // The row stays in the agent-status store, idle and no longer running here, and a session + // list that connects now still gets it. + expect(rig.sink.forget).not.toHaveBeenCalled() + expect(rig.sink.publish.mock.calls.at(-1)?.[0]).toMatchObject({ sessionId: SESSION }) + expect(statusRows().at(-1)?.hostExecutionOwned).toBeUndefined() + const late: { type: string; sessions?: AgentSessionStatusSummary[] }[] = [] + rig.host.subscribeStatus({ id: 'late', emit: (event) => late.push(event) }) + expect(late[0]?.sessions?.map((summary) => summary.sessionId)).toContain(SESSION) + expect(reader.events.some((event) => event.type === 'end')).toBe(false) + + const sent = await rig.host.send(CALLER, restTestSend('after the close', fence())) + expect(sent.ok).toBe(true) + // A cold start: reopen, reconcile and acquire. + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2), COLD_START) + // The reader opened on the first handle is fed by the second. + await vi.waitFor(() => expect(readerSaw(reader.events).texts).toContain('after the close')) + }) + + it('drops the agent-status row of a chat whose tab is gone (P2-14, retired)', async () => { + await foundRestTestChat(rig) + await rig.store.setSessionTabVisibility(SESSION, false) + rig.clock.now += IDLE_MS + 1 + + await sweepOnce(rig.host) + expect(rig.host.hasSession(SESSION)).toBe(false) + expect(rig.sink.forget).toHaveBeenCalled() + }) + + it('forgets the row of a resting chat once its tab closes (P2-32)', async () => { + await foundRestTestChat(rig) + rig.clock.now += IDLE_MS + 1 + await sweepOnce(rig.host) + expect(rig.sink.forget).not.toHaveBeenCalled() + + await rig.host.setSessionTabVisibility(SESSION, false) + expect(rig.sink.forget).toHaveBeenCalled() + }) + + it('keeps the row when a chat with an open tab is evicted, and forgets it once the tab closes', async () => { + await foundRestTestChat(rig) + + await rig.host.close(SESSION) + expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION) + // The stop says not-running; the row belongs to the tab, so nothing forgets it. + expect(rig.sink.forget).not.toHaveBeenCalled() + expect(rig.sink.publish.mock.calls.at(-1)?.[0]).toMatchObject({ sessionId: SESSION }) + expect(rig.sink.publish.mock.calls.at(-1)?.[0].hostExecutionOwned).toBeUndefined() + + await rig.host.setSessionTabVisibility(SESSION, false) + expect(rig.sink.forget).toHaveBeenCalledOnce() + }) + + it('never hands a reader a handle it is closing (P2-25)', async () => { + await foundRestTestChat(rig) + const journal = openSession()?.journal + if (!journal) { + throw new Error('the conversation should be open') + } + const closing = Promise.withResolvers() + const gate = Promise.withResolvers() + const close = journal.close.bind(journal) + vi.spyOn(journal, 'close').mockImplementation(async () => { + await close() + closing.resolve() + await gate.promise + }) + rig.clock.now += IDLE_MS + 1 + const tick = sweepOnce(rig.host) + await closing.promise + + const history = rig.host.history({ sessionId: SESSION, direction: 'tail' }) + const facts = readStructuredSessionGateFacts(SESSION) + gate.resolve() + await tick + + const page = await history + expect(page.ok && page.page.items.length).toBeGreaterThan(0) + await expect(facts).resolves.not.toBeNull() + // And the handle a write finds is a live one. + const sent = await rig.host.send(CALLER, restTestSend('after the close', fence())) + expect(sent.ok).toBe(true) + // A cold start: reopen, reconcile and acquire. + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2), COLD_START) + }) + + it('never re-enters the session lock: a send and a Stop right after still finish (P2-28)', async () => { + await foundRestTestChat(rig) + rig.clock.now += IDLE_MS + 1 + const tick = sweepOnce(rig.host) + const within = (work: Promise) => + Promise.race([ + work, + new Promise((_, reject) => setTimeout(() => reject(new Error('wedged')), 2_000)) + ]) + const sent = within(rig.host.send(CALLER, restTestSend('right after', fence()))) + + await within(tick) + expect(await sent).toMatchObject({ ok: true }) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2)) + await expect(within(sweepOnce(rig.host))).resolves.toBeUndefined() + }) +}) + +describe('the sweep and the lease (P2-20)', () => { + it('reads only open conversations and never probes or resolves a lease', async () => { + await foundRestTestChat(rig) + const probeOwner = vi.fn(async () => ({ outcome: 'pid-absent' as const })) + await rig.restart({ probeOwner }) + setStructuredAgentSessionHost(rig.host) + await rig.store.transitionHandoff(SESSION, (current) => ({ + ...current, + lease: { ...current.lease, handoffStage: 'recovering' } + })) + const resolveRecovery = vi.spyOn( + rig.host.collaboratorsForTests().runtimeState, + 'resolveRecovery' + ) + // Open, at rest, on a lease still recovering. + await rig.host.journalSnapshot(SESSION) + expect(rig.host.hasSession(SESSION)).toBe(true) + rig.clock.now += IDLE_MS + 1 + + await sweepOnce(rig.host) + expect(rig.host.hasSession(SESSION)).toBe(false) + expect(probeOwner).not.toHaveBeenCalled() + expect(resolveRecovery).not.toHaveBeenCalled() + }) +}) + +describe('a start that never finishes (P2-15)', () => { + it('is stopped by the sweep, and the delivery loop alone writes its one row and rejection', async () => { + const stopReason = 'Codex never finished starting, so Orca stopped it.' + const order: string[] = [] + const started = Promise.withResolvers() + rig.adapter.closeSession.mockImplementation(async () => { + order.push('stopped') + started.resolve() + return true + }) + const spawn = rig.adapter.acquire.getMockImplementation()! + rig.adapter.acquire.mockImplementationOnce(async (input) => ({ + ...(await spawn(input)), + providerChildPhase: 'starting' as const + })) + Object.assign(rig.host.deps.adapter, { awaitStarted: () => started.promise }) + const reject = AgentSessionJournal.prototype.rejectQueuedSubmissions + vi.spyOn(AgentSessionJournal.prototype, 'rejectQueuedSubmissions').mockImplementation(function ( + this: AgentSessionJournal, + ...args + ) { + // Not the open's sweep of an earlier process's leftovers. + if (args[1] !== DISPATCH_REJECTED_HOST_RESTARTED) { + order.push(`rejected: ${args[1]}`) + } + return reject.apply(this, args) + }) + const reader = collectSubscriber() + const attached = await rig.host.attach(CALLER, hostTestAttachParams(null)) + expect(attached.ok).toBe(true) + await rig.host.subscribe({ id: 'reader', sessionId: SESSION, emit: reader.emit }) + const sent = await rig.host.send(CALLER, restTestSend('stuck behind the start', fence())) + expect(sent.ok).toBe(true) + rig.clock.now += IDLE_MS + 1 + + await sweepOnce(rig.host) + expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION) + await vi.waitFor(() => + expect(readerSaw(reader.events).submissions).toContainEqual( + expect.objectContaining({ dispatchState: 'rejected', reason: stopReason }) + ) + ) + // One rejection, written after the stop by the loop, with the stop's own words. + expect(order).toEqual(['stopped', `rejected: ${stopReason}`]) + const errorRows = reader.events.flatMap((event) => + event.type === 'batch' + ? event.batch.items.filter( + (item) => item.body.kind === 'status' && item.body.tone === 'error' + ) + : [] + ) + expect(errorRows.map((item) => (item.body.kind === 'status' ? item.body.text : null))).toEqual([ + stopReason + ]) + expect(rig.adapter.dispatch).not.toHaveBeenCalled() + + const again = await rig.host.send(CALLER, restTestSend('try again', fence())) + expect(again.ok).toBe(true) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce(), COLD_START) + await expect(sweepOnce(rig.host)).resolves.toBeUndefined() + }) +}) + +describe('the wind-down retry with a message queued (P2-31)', () => { + it('waits for the delivery rather than rejecting a message accepted after the failed stop', async () => { + const queued = { clientMessageId: 'm', dispatchState: 'pending', handoverRecorded: true } + const session = { + journal: { + submissions: () => [queued], + pendingSubmissions: () => [queued], + snapshot: () => ({ items: [] }) + }, + child: null, + owesProviderChildWindDown: { generation: 'generation-1', fence: 1 } + } + const stopAgent = vi.fn(async () => undefined) + const sweep = new StructuredAgentSessionIdleSweep({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a session fixture carrying only the journal and child facts the sweep reads. + sessions: Object.assign(new Map([[SESSION, session as never]]), { + lastActivityAt: () => 0, + touch: () => undefined + }), + serialize: (_id, task) => task(), + now: () => IDLE_MS + 1, + isDisposed: () => false, + deliveryActive: () => true, + backgroundTaskState: () => undefined, + hasOpenDispatch: () => false, + stopAgent, + stopStartingAgent: stopAgent, + closeConversation: vi.fn(async () => false), + onError: (_id, error) => { + throw error + } + }) + await sweep.tick() + expect(stopAgent).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts new file mode 100644 index 000000000000..73d1825bee5b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -0,0 +1,146 @@ +// How a conversation is reached, closed and swept — the host's lifetime rules in one place. +// +// A conversation is reached only through `conversation`, which opens it at rest and starts no +// agent. It closes when its tab closes or the idle sweep finds its handle is only a cache. Every +// public entry point here takes the session's serialize once and calls the under-serialize forms, +// because the queue is not reentrant. + +import { TUI_AGENT_DISPLAY_NAMES } from '../../../shared/tui-agent-display-names' +import type { StructuredAgentSessionConversations } from './structured-agent-session-conversations' +import { + abandonQueuedStructuredAgentSessionMessages, + closeStructuredAgentSessionConversationUnderSerialize, + stopStructuredAgentSessionAgentUnderSerialize, + type StructuredAgentSessionLifetimeContext +} from './structured-agent-session-host-lifetime' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { StructuredAgentSessionIdleSweep } from './structured-agent-session-idle-sweep' +import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' +import { adapterSupportsRecord } from './structured-agent-session-provider-support' +import { isUnusableSqliteDatabaseError } from '../../sqlite/sqlite-read-failure' +import type { StructuredAgentSessionStartupStep } from './structured-agent-session-restart-restore' + +export type StructuredAgentSessionConversationLifetime = ReturnType< + typeof createStructuredAgentSessionConversationLifetime +> + +export function createStructuredAgentSessionConversationLifetime(host: { + /** Resolved per call: the host's collaborators are assigned after this is built. */ + context: () => StructuredAgentSessionLifetimeContext + sessions: StructuredAgentSessionConversations + serialize: (sessionId: string, task: () => Promise) => Promise + /** PR 1's one open function, for a caller inside the session's serialize. */ + open: (sessionId: string) => Promise + deliveryActive: (sessionId: string) => boolean + /** The handle closed: `listed` keeps the chat's row in the agent-status store for its tab. */ + closeStatus: (sessionId: string, options: { listed: boolean }) => void +}) { + let disposed = false + const { sessions, serialize } = host + const deps = () => host.context().deps + // The sweep's stop puts an idle agent to rest: nothing is queued, so no loop reads its cause. + const stopAgent = (sessionId: string) => + stopStructuredAgentSessionAgentUnderSerialize(host.context(), sessionId) + + const closeConversation = (sessionId: string): Promise => + closeStructuredAgentSessionConversationUnderSerialize( + { + sessions, + closeStatus: (id) => + host.closeStatus(id, { listed: deps().store.listVisibleSessionIds().includes(id) }) + }, + sessionId + ) + + const idleSweep = new StructuredAgentSessionIdleSweep({ + sessions, + serialize, + now: () => host.context().now(), + isDisposed: () => disposed, + deliveryActive: host.deliveryActive, + backgroundTaskState: (sessionId) => deps().adapter.backgroundTaskState?.(sessionId), + hasOpenDispatch: (sessionId) => { + const record = deps().store.getRecord(sessionId) + return record !== null && deps().hasOpenDispatch?.(record) === true + }, + stopAgent, + // A host stop with its reason: the delivery loop waiting on this child writes the one error + // row and rejects what is queued with it. + stopStartingAgent: (sessionId) => + stopStructuredAgentSessionAgentUnderSerialize(host.context(), sessionId, { + cause: 'host-stop', + reason: `${TUI_AGENT_DISPLAY_NAMES[sessions.get(sessionId)?.params.provider ?? 'claude']} never finished starting, so Orca stopped it.` + }), + closeConversation, + onError: (sessionId, error) => deps().onEventSinkError?.({ sessionId, error }), + ...deps().idleSweep + }) + + return { + idleSweep, + stopAgent, + /** Quit has begun: nothing opens a conversation or sweeps one after this. */ + dispose: (): void => { + disposed = true + idleSweep.dispose() + }, + /** + * The only way any code reaches a session. An open conversation answers without the lock, so + * a read never waits behind a start; a closed one is opened once, under it. Nothing here + * touches the lease or starts a child. Use the result before the next `await`: a close can + * drop it after. + */ + conversation: async (sessionId: string): Promise => { + const open = sessions.get(sessionId) + if (open) { + return open + } + const record = deps().store.getRecord(sessionId) + if (!record) { + throw new Error('agent_session_identity_required') + } + if (!adapterSupportsRecord(deps().adapter, record)) { + throw new Error('structured_agent_session_unsupported') + } + return serialize(sessionId, async () => { + // Read at the open itself: a read queued before quit began runs after it. + if (disposed) { + throw new Error(AGENT_SESSION_NOT_ATTACHED.code) + } + const session = await host.open(sessionId).catch((error: unknown) => { + // Final: the same file answers the same on every retry. Anything else stays retryable. + throw isUnusableSqliteDatabaseError(error) + ? new Error('agent_session_journal_unreadable') + : error + }) + if (!session) { + throw new Error('agent_session_identity_required') + } + return session + }) + }, + isDisposed: (): boolean => disposed, + /** The startup pass's per-chat step, under the session's lock and never after quit began. A + * conversation the step opens only to derive status closes again as the idle sweep's would. */ + startupStep: ((sessionId, step) => + serialize(sessionId, async () => + disposed + ? undefined + : step({ session: sessions.get(sessionId), close: () => closeConversation(sessionId) }) + )) satisfies StructuredAgentSessionStartupStep, + /** Ends a chat's resources, not the chat: its record and journal stay on disk, and what is + * still queued will not be sent. */ + close: (sessionId: string): Promise => + serialize(sessionId, async () => { + const session = sessions.get(sessionId) + if (session) { + // Abandoned before the stop, so no start delivers it. + await abandonQueuedStructuredAgentSessionMessages(deps(), sessionId, session.journal) + } + await stopStructuredAgentSessionAgentUnderSerialize(host.context(), sessionId, { + cause: 'evict' + }) + await closeConversation(sessionId) + }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts new file mode 100644 index 000000000000..ceaa951e4d75 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -0,0 +1,134 @@ +// The one way a conversation's journal becomes open on this host: for a send, for a reader, and +// for an attach that finds none open. +// +// It opens with recovery, so an unusable journal is rebuilt rather than refused, and it marks what +// an earlier host process handed over and left unanswered as in doubt — the crash boundary. That +// needs no lease: provider history decides such a row later, under a won lease, in the attach. A +// row an earlier process accepted and never handed over is the delivery loop's, which the open +// wakes. Nothing here starts a provider child. + +import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + attachFingerprintFields, + journalIdentityFor, + type AgentSessionAttachParams +} from './structured-agent-session-attach' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' + +export type OpenedStructuredAgentSessionConversation = { + session: StructuredAgentSessionHostSession + /** Set when the journal was rebuilt on the way; readers reload from a snapshot. */ + reset: AgentJournalResetReason | null +} + +export type StructuredAgentSessionConversationOpenDeps = { + store: Pick + adapter: Pick + journalRoot: string + onEventSinkError?: StructuredAgentSessionHostDeps['onEventSinkError'] +} + +export type StructuredAgentSessionConversationOpenContext = { + deps: StructuredAgentSessionConversationOpenDeps + sessions: Map + /** Indexes a conversation that just became open; the host publishes it and wakes delivery. */ + adoptOpened: ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ) => Promise +} + +/** The open conversation, or null when this host has no record of it. For a caller inside the + * session's serialize, which is what makes "not open yet" exact. */ +export async function openStructuredAgentSessionConversation( + context: StructuredAgentSessionConversationOpenContext, + sessionId: string +): Promise { + const open = context.sessions.get(sessionId) + if (open) { + return open + } + const record = context.deps.store.getRecord(sessionId) + if (!record) { + return null + } + const opened = await openStructuredAgentSessionConversationJournal(context.deps, record) + await context.adoptOpened(sessionId, opened) + return opened.session +} + +/** The open itself, indexed by nobody yet: the caller adopts the result. */ +export async function openStructuredAgentSessionConversationJournal( + deps: Omit, + record: AgentSessionRecord +): Promise { + const { sessionId } = record + const fence = record.lease.runtimeFence + const params = attachParamsForRecord(record, { + clientOperationId: `read-restore:${sessionId}`, + expectedRuntimeFence: fence + }) + const identity = journalIdentityFor(record, params) + const opened = await openAgentSessionJournalWithRecovery({ + identity, + journalDir: journalDirectoryFor(deps.journalRoot, { + workspaceId: record.location.workspaceId, + sessionId + }), + fence, + historyFilePath: (await deps.adapter.historyFilePath?.({ identity })) ?? null + }) + try { + // A queued row found here is a leftover the delivery loop's first step rejects; a handed-over + // one is only doubt, which provider history decides under a won lease. + await opened.journal.markPendingSubmissionsUnknown(fence) + } catch (error) { + deps.onEventSinkError?.({ sessionId, error }) + } + return { + session: { journal: opened.journal, params, child: null }, + reset: opened.recovery?.reset ?? null + } +} + +export function attachParamsForRecord( + record: AgentSessionRecord, + input: { + clientOperationId: string + expectedRuntimeFence: number + } +): AgentSessionAttachParams { + const params: AgentSessionAttachParams = { + envelope: { + sessionId: record.sessionId, + clientOperationId: input.clientOperationId, + expectedRuntimeFence: input.expectedRuntimeFence, + payloadFingerprint: '' + }, + location: record.location, + provider: record.provider, + agent: record.provider, + accountHome: record.accountHome, + runtimeKind: 'native' + } + return { + ...params, + envelope: { + ...params.envelope, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.attach', + sessionId: record.sessionId, + fields: attachFingerprintFields(params) + }) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts index 202ffe50ae4d..b294540990d9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts @@ -37,10 +37,7 @@ function session(journal: AgentSessionJournal) { return { journal, params: hostTestAttachParams(null), - fence: 0, - hasProviderChild: false, - providerChildPhase: 'ready' as const, - acquisitionGeneration: null + child: null } } @@ -57,7 +54,8 @@ describe('a conversation delivers what its journal commits', () => { const deliver = vi.fn() const conversations = new StructuredAgentSessionConversations({ deliver, - onDeliveryError: vi.fn() + onDeliveryError: vi.fn(), + now: () => 0 }) const journal = await openJournal('a') conversations.set('session-1', session(journal)) @@ -73,7 +71,11 @@ describe('a conversation delivers what its journal commits', () => { const deliver = vi.fn() // Collaborators hold the host's map as a plain `Map`; `set` still reaches the binding. const sessions: Map = - new StructuredAgentSessionConversations({ deliver, onDeliveryError: vi.fn() }) + new StructuredAgentSessionConversations({ + deliver, + onDeliveryError: vi.fn(), + now: () => 0 + }) const journal = await openJournal('a') sessions.set('session-1', session(journal)) @@ -86,7 +88,8 @@ describe('a conversation delivers what its journal commits', () => { const deliver = vi.fn() const conversations = new StructuredAgentSessionConversations({ deliver, - onDeliveryError: vi.fn() + onDeliveryError: vi.fn(), + now: () => 0 }) const journal = await openJournal('a') conversations.set('session-1', session(journal)) @@ -100,7 +103,8 @@ describe('a conversation delivers what its journal commits', () => { const deliver = vi.fn() const conversations = new StructuredAgentSessionConversations({ deliver, - onDeliveryError: vi.fn() + onDeliveryError: vi.fn(), + now: () => 0 }) const replaced = await openJournal('a') const current = await openJournal('b') @@ -118,7 +122,8 @@ describe('a conversation delivers what its journal commits', () => { const deliver = vi.fn() const conversations = new StructuredAgentSessionConversations({ deliver, - onDeliveryError: vi.fn() + onDeliveryError: vi.fn(), + now: () => 0 }) const journal = await openJournal('a') conversations.set('session-1', session(journal)) @@ -136,7 +141,8 @@ describe('a conversation delivers what its journal commits', () => { deliver: () => { throw failure }, - onDeliveryError + onDeliveryError, + now: () => 0 }) const journal = await openJournal('a') conversations.set('session-1', session(journal)) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts index 2f38ddf4da25..6fcaef07483a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts @@ -8,15 +8,21 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-sessi * * Delivery runs a microtask after the commit, so a reader that throws cannot fail a write that is * already durable. A handle this map has since replaced or dropped delivers nothing. + * + * Each entry also carries when it last saw activity — its open, then every journal publish — which + * is the idle sweep's clock. In memory only, so it dies with the entry. */ export class StructuredAgentSessionConversations extends Map< string, StructuredAgentSessionHostSession > { + private readonly activity = new Map() + constructor( private readonly delivery: { deliver: (sessionId: string, journal: AgentSessionJournal) => void onDeliveryError: (sessionId: string, error: unknown) => void + now: () => number } ) { super() @@ -42,6 +48,22 @@ export class StructuredAgentSessionConversations extends Map< } }) }) + this.activity.set(sessionId, this.delivery.now()) return super.set(sessionId, session) } + + override delete(sessionId: string): boolean { + this.activity.delete(sessionId) + return super.delete(sessionId) + } + + touch(sessionId: string): void { + if (this.has(sessionId)) { + this.activity.set(sessionId, this.delivery.now()) + } + } + + lastActivityAt(sessionId: string): number | undefined { + return this.activity.get(sessionId) + } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts index d553fbaecf37..8ad51871c240 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts @@ -190,6 +190,7 @@ describe('dead structured-session generation settlement', () => { | 'submissions' | 'markPendingSubmissionsUnknown' | 'rejectPendingSubmissions' + | 'rejectQueuedSubmissions' | 'appendLifecycleBatch' > = { snapshot: () => ({ @@ -203,6 +204,9 @@ describe('dead structured-session generation settlement', () => { rejectPendingSubmissions: async () => { throw new Error('journal_closed') }, + rejectQueuedSubmissions: async () => { + throw new Error('journal_closed') + }, appendLifecycleBatch: async () => { throw new Error('journal_closed') } @@ -273,7 +277,7 @@ describe('dead structured-session generation settlement', () => { pendingSubmissionReason: 'provider_closed_before_acknowledgement', verdict: { state: 'interrupted', completedAt: 1_000 }, unexpectedExitReason: 'claude stream-json exited (code 1): not signed in', - exitedDuringStartup: true + exitedDuringStartup: { generation: 'generation-1' } }) const reason = diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 1b5f73fdbdae..f79cbfcbc6bd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -11,6 +11,7 @@ import { cancelledJournalPromptBody } from '../agent-session-journal/journal-prompt-body-bounds' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' import { runningTurnLifecycleRevisions, type StructuredAgentSessionTurnVerdict @@ -55,6 +56,14 @@ export function providerStartupFailureOutcome(reason?: string): string { : 'The provider stopped before it finished starting.' } +/** Why a message accepted but not yet handed over was rejected when its child exited. */ +export function providerExitBeforeDeliveryRejection(reason?: string): string { + const detail = exitReasonDetail(reason) + return detail + ? `The provider stopped before this message was sent: ${detail}.` + : 'The provider stopped before this message was sent.' +} + /** Why a send a child that never started left unwritten was rejected. The child's own diagnostic is * the cause the user can act on, so it is the reason, in the words the chat row uses. */ export function providerStartupFailureRejection(cause?: unknown): string { @@ -72,7 +81,7 @@ function exitReasonDetail(reason: string | undefined): string | undefined { type DeadGenerationSubmission = Pick< ReturnType[number], - 'clientMessageId' | 'dispatchState' | 'recovered' + 'clientMessageId' | 'dispatchState' | 'recovered' | 'handoverRecorded' | 'handedOverAt' > export type DeadGenerationJournal = { @@ -142,8 +151,9 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { showUnexpectedExitOutcome?: boolean /** Why the provider stopped, when the host has it. Rendered with the outcome copy. */ unexpectedExitReason?: string - /** The provider never finished starting; the outcome says so instead of naming a response. */ - exitedDuringStartup?: boolean + /** The provider never finished starting: the start that failed, keyed by the child's + * generation. Its row is the one the delivery loop writes for the same start. */ + exitedDuringStartup?: { generation: string | null } onError?: (sessionId: string, error: unknown) => void }): Promise { try { @@ -152,9 +162,10 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { return true } - // A child that never proved its start accepted nothing — input is written only after it - // initializes — so every send it left unanswered is provably unwritten and is rejected with the - // child's own diagnostic. A proven child's unanswered sends stay in doubt. + // A queued message is the delivery loop's to settle: it was never handed to this child. A + // child that never proved its start accepted nothing either — input is written only after it + // initializes — so every send it was handed is rejected with the child's own diagnostic. A + // proven child's handed-over sends stay in doubt. await (input.exitedDuringStartup ? input.journal.rejectPendingSubmissions( input.fence, @@ -163,17 +174,21 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { : input.journal.markPendingSubmissionsUnknown(input.fence, input.pendingSubmissionReason)) const items = input.journal.snapshot().items const mutations: JournalLifecycleMutationInput[] = [] - if (showUnexpectedExitOutcome) { + if (showUnexpectedExitOutcome && input.exitedDuringStartup) { + // Until views stop starting children, a start can die with nothing queued for the loop. + mutations.push( + structuredAgentSessionStartFailureRow( + input.exitedDuringStartup.generation ?? input.settlementId, + providerStartupFailureOutcome(input.unexpectedExitReason) + ) + ) + } else if (showUnexpectedExitOutcome) { mutations.push({ kind: 'item', identity: { provider: 'orca', clientMessageId: input.settlementId }, body: { kind: 'status', - text: boundJournalStatusText( - input.exitedDuringStartup - ? providerStartupFailureOutcome(input.unexpectedExitReason) - : unexpectedProviderExitOutcome(input.unexpectedExitReason) - ) + text: boundJournalStatusText(unexpectedProviderExitOutcome(input.unexpectedExitReason)) } }) } @@ -247,7 +262,9 @@ function hasUnsettledSubmission(journal: DeadGenerationJournal): boolean { return submissions ? submissions.some( (submission) => - submission.dispatchState === 'pending' || + // A queued message is not work in progress: nothing has it yet. + (submission.dispatchState === 'pending' && + !(submission.handoverRecorded && submission.handedOverAt === undefined)) || (submission.dispatchState === 'unknown' && submission.recovered !== true) ) : (journal.pendingSubmissions?.().length ?? 0) > 0 diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts new file mode 100644 index 000000000000..3c3ddbb82efb --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -0,0 +1,253 @@ +// The one thing that starts a provider child for a send, the one thing that hands a message to +// it, and the one thing that settles a queued message because of a start, a child or a leftover. +// +// A send is accepted on its own serialized step and returns; this loop does the rest. It exists +// for a session exactly while a message is queued there — accepted, not yet handed over — and +// every step re-reads the journal and the conversation's child record to decide, so there is no +// loop state to disagree with them. Each step is its own serialized task. That is what lets a Stop +// that arrives while a start holds the queue withdraw the queued messages before the handover that +// would have written them. Stop and the conversation's close are the only other writers of a +// queued message: a child's exit only ends the child, and this loop reads why. + +import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { DISPATCH_REJECTED_HOST_RESTARTED } from '../../../shared/structured-agent-session-dispatch-rejection' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { + providerExitBeforeDeliveryRejection, + providerStartupFailureOutcome +} from './structured-agent-session-dead-generation-settlement' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-agent-start' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' +import { + oldestQueuedSubmission, + recordStructuredAgentSessionStartFailure +} from './structured-agent-session-start-failure-row' +import { handOverSubmission } from './structured-agent-session-turns' + +export type StructuredAgentSessionDeliveryLoopDeps = { + sessions: ReadonlyMap + adapter: StructuredAgentSessionAdapter + serialize: (sessionId: string, task: () => Promise) => Promise + /** A start step, tracked from enqueue so quit waits for the child it may produce. */ + trackStart: (start: Promise) => Promise + /** Gives the session a provider child if it has none; for a caller inside `serialize`. */ + /** Starts a child for `startedFor`, the queued message at the head, if the session has none. */ + ensureProviderChild: ( + sessionId: string, + startedFor: string + ) => Promise + /** The fence the conversation's own writes carry; see `structuredAgentSessionConversationFence`. */ + conversationFence: (sessionId: string) => number + /** What the chat says when the session could not be made ready. */ + startFailureText: (sessionId: string, cause: AgentSessionWireRefusal) => string + onError: (sessionId: string, error: unknown) => void +} + +type Step = 'continue' | 'stop' + +type Prepared = + | 'stop' + | { ok: false; refusal: AgentSessionWireRefusal } + | { ok: true; awaited: StructuredAgentSessionProviderChildIdentity | null } + +type StartFailure = { startKey: string | null; text: string } + +export class StructuredAgentSessionDeliveryLoop { + private readonly running = new Set() + private disposed = false + + constructor(private readonly deps: StructuredAgentSessionDeliveryLoopDeps) {} + + isRunning(sessionId: string): boolean { + return this.running.has(sessionId) + } + + /** Quit: no step after this one starts a child or hands a message over. */ + dispose(): void { + this.disposed = true + } + + /** From inside the session's serialize, after a message was accepted or the conversation + * opened. A loop already running re-reads the journal on its next step. */ + wake(sessionId: string): void { + if (this.disposed || this.running.has(sessionId)) { + return + } + this.running.add(sessionId) + void this.run(sessionId) + } + + private async run(sessionId: string): Promise { + try { + for (;;) { + const prepared = await this.deps.trackStart( + this.deps.serialize(sessionId, () => this.prepare(sessionId)) + ) + if (prepared === 'stop') { + return + } + if (!prepared.ok) { + const text = this.deps.startFailureText(sessionId, prepared.refusal) + await this.deps.serialize(sessionId, () => this.fail(sessionId, { startKey: null, text })) + return + } + // A child published before it proved its start takes no input yet; waited for outside + // the queue so a Stop can reach it meanwhile. + const failure = await this.deps.adapter.awaitStarted?.(sessionId) + const handed = await this.deps.serialize(sessionId, () => + this.handOver(sessionId, prepared.awaited, failure || null) + ) + if (handed === 'stop') { + return + } + } + } catch (error) { + this.deps.onError(sessionId, error) + const text = this.deps.startFailureText(sessionId, { + code: 'agent_session_owner_restart_failed', + message: error instanceof Error ? error.message : String(error) + }) + await this.deps + .serialize(sessionId, () => this.fail(sessionId, { startKey: null, text })) + .catch((failure: unknown) => { + // Rows left queued are rejected by the next open, or by the next loop an accept wakes. + this.running.delete(sessionId) + this.deps.onError(sessionId, failure) + }) + } + } + + /** Settles what an earlier host process left queued, then makes the session ready. */ + private async prepare(sessionId: string): Promise { + const session = this.deps.sessions.get(sessionId) + if (!session || this.disposed) { + return this.stop(sessionId) + } + await session.journal.rejectQueuedSubmissions( + this.deps.conversationFence(sessionId), + DISPATCH_REJECTED_HOST_RESTARTED, + // A handle closes only with nothing queued, so one an earlier handle wrote is a leftover. + (submission) => session.journal.wroteBeforeOpen(submission.acceptedSequence) + ) + const oldest = oldestQueuedSubmission(session) + if (!oldest) { + return this.stop(sessionId) + } + const failedStart = startThatFailedWhileQueued(session, oldest) + if (failedStart) { + return this.fail(sessionId, failedStart) + } + const ready = await this.deps.ensureProviderChild(sessionId, oldest.clientMessageId) + if (!ready.ok) { + return ready + } + const child = this.deps.sessions.get(sessionId)?.child + // The child this run waits on; handover checks it is still the one there. + return { + ok: true, + awaited: child ? { generation: child.generation, fence: child.fence } : null + } + } + + private async handOver( + sessionId: string, + awaited: StructuredAgentSessionProviderChildIdentity | null, + startFailure: string | null + ): Promise { + const session = this.deps.sessions.get(sessionId) + if (!session || this.disposed) { + return this.stop(sessionId) + } + // Re-derived here, not carried from the start: the child may have ended, or another may have + // taken its place, since. + const { child } = session + const awaitedChild = + child && awaited && child.generation === awaited.generation && child.fence === awaited.fence + ? child + : null + // The host's `starting` trails the adapter's `started` by one serialized step, so for the child + // waited on, the adapter's own answer decides whether its start landed. + if (!awaitedChild || (awaitedChild.phase === 'starting' && startFailure !== null)) { + // The child waited on is gone, replaced by another, or settled its start without proving it. + const ended = awaitedChild ? undefined : session.lastEndedChild + // A user's Stop is not a failure: the next step starts, or waits on, a child for what is + // queued. A host stop is: its cause is why the start did not land. + if (ended?.cause === 'user-stop') { + return 'continue' + } + return this.fail(sessionId, { + startKey: awaited?.generation ?? null, + text: ended ? endedChildRejection(ended) : (startFailure ?? providerStartupFailureOutcome()) + }) + } + const next = oldestQueuedSubmission(session) + if (!next) { + return this.stop(sessionId) + } + await handOverSubmission( + { + sessionId, + journal: session.journal, + fence: awaitedChild.fence, + adapter: this.deps.adapter, + providerChildPhase: () => this.deps.sessions.get(sessionId)?.child?.phase + }, + next + ) + return 'continue' + } + + private async fail(sessionId: string, failure: StartFailure): Promise<'stop'> { + const session = this.deps.sessions.get(sessionId) + if (session) { + await recordStructuredAgentSessionStartFailure( + { journal: session.journal, fence: this.deps.conversationFence(sessionId) }, + failure + ) + } + return this.stop(sessionId) + } + + /** Inside the serialized step that found nothing to do, so an accept after it wakes anew. */ + private stop(sessionId: string): 'stop' { + this.running.delete(sessionId) + return 'stop' + } +} + +/** A start that died while this message waited on it — a view's, say — is the message's failed + * start: settled with it, under its key, rather than started again into the same failure. */ +function startThatFailedWhileQueued( + session: StructuredAgentSessionHostSession, + oldest: NonNullable> +): StartFailure | null { + const ended = session.lastEndedChild + if ( + session.child || + !ended?.duringStartup || + ended.cause === 'user-stop' || + oldest.acceptedSequence === undefined || + ended.endedAt.epoch !== session.journal.cursor().epoch || + ended.endedAt.sequence < oldest.acceptedSequence + ) { + return null + } + return { startKey: ended.generation, text: endedChildRejection(ended) } +} + +const HOST_STOPPED_BEFORE_DELIVERY = 'Orca stopped the agent before this message was sent.' + +/** Why a queued message the child never took is rejected, in the words the chat row uses. */ +function endedChildRejection(ended: StructuredAgentSessionEndedChild): string { + if (ended.cause === 'host-stop') { + return ended.reason ?? HOST_STOPPED_BEFORE_DELIVERY + } + const reason = ended.reason ?? undefined + return ended.duringStartup + ? providerStartupFailureOutcome(reason) + : providerExitBeforeDeliveryRejection(reason) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 19989e107119..0c2a76e05a57 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -7,12 +7,8 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' -import type { StructuredAgentSessionHolds } from './structured-agent-session-holds' import { settleStructuredAgentSessionProviderStarted } from './structured-agent-session-provider-started' -import { - isStructuredAgentSessionRecoveryTicketCurrent, - settleUnexpectedStructuredAgentSessionExit -} from './structured-agent-session-unexpected-exit' +import { settleUnexpectedStructuredAgentSessionExit } from './structured-agent-session-unexpected-exit' export class StructuredAgentSessionEventRecovery { private readonly sinkFailures = new Set() @@ -25,12 +21,8 @@ export class StructuredAgentSessionEventRecovery { flushLifecycle: (sessionId: string) => Promise publishFence: (sessionId: string, session: StructuredAgentSessionHostSession) => void publishStatus?: (sessionId: string) => void - hasResumeCapableHolder: (sessionId: string) => boolean - restartReleaseGrace: (sessionId: string) => void serialize: (sessionId: string, task: () => Promise) => Promise now: () => number - /** The one restart every asker shares; the holds put an unheld child on the idle clock. */ - ensureProviderChild: StructuredAgentSessionHolds['ensureProviderChild'] onBarrierError: (sessionId: string, error: unknown) => void } ) {} @@ -42,14 +34,13 @@ export class StructuredAgentSessionEventRecovery { this.sinkFailures.add(sessionId) void this.context .serialize(sessionId, async () => { - const session = this.context.sessions.get(sessionId) + const child = this.context.sessions.get(sessionId)?.child const stop = this.context.deps.adapter.forceCloseSession ?? this.context.deps.adapter.closeSession - if (!session?.hasProviderChild || !stop) { + if (!child || !stop) { return null } - const fence = session.fence - const acquisitionGeneration = session.acquisitionGeneration + const { fence, generation: acquisitionGeneration } = child const stopped = await stopAgentSessionProviderRoot(() => stop(sessionId)) if (!stopped || !acquisitionGeneration) { return null @@ -68,34 +59,12 @@ export class StructuredAgentSessionEventRecovery { .finally(() => this.sinkFailures.delete(sessionId)) } + /** An exit is settled and shown; nothing restarts the child. The next send does, through the + * delivery loop, which also owns any message still queued. */ async handle(event: StructuredAgentSessionLifecycleEvent): Promise { if (event.type === 'started') { return settleStructuredAgentSessionProviderStarted(this.context, event) } - const ticket = await settleUnexpectedStructuredAgentSessionExit(this.context, event) - if (!ticket) { - return - } - // One serialized step with the ticket check inside it: a hold or a send that got there first - // has already replaced the owner, and this step finds that child and attaches nothing — or, - // once the lease has moved on, refuses on the stale ticket rather than spawning a second child. - try { - const resumed = await this.context.serialize(ticket.sessionId, () => - this.context.ensureProviderChild(ticket.sessionId, { - admitRecoveryTicket: () => - isStructuredAgentSessionRecoveryTicketCurrent(this.context, ticket) - }) - ) - if (!resumed.ok && isStructuredAgentSessionRecoveryTicketCurrent(this.context, ticket)) { - this.context.onBarrierError( - ticket.sessionId, - new Error(`${resumed.refusal.code}: ${resumed.refusal.message}`) - ) - } - } catch (error) { - if (isStructuredAgentSessionRecoveryTicketCurrent(this.context, ticket)) { - this.context.onBarrierError(ticket.sessionId, error) - } - } + await settleUnexpectedStructuredAgentSessionExit(this.context, event) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts index cb0cb2fb2334..8fc810de2a4d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts @@ -30,8 +30,8 @@ function context(): StructuredAgentSessionEvictionContext & { order: string[] } return true }) } as unknown as StructuredAgentSessionEvictionContext['adapter'], - forget: vi.fn(async () => { - order.push('forget') + acknowledgeRelease: vi.fn(() => { + order.push('acknowledgeRelease') }), discardSink: vi.fn(() => order.push('discardSink')), settleWork: vi.fn(async () => { @@ -51,7 +51,7 @@ function runtimeState(): StructuredAgentSessionHostRuntimeState { } describe('structured agent session eviction', () => { - it('stops the child before it lets the sink go, then forgets the session', async () => { + it('stops the child before it lets the sink go, then acknowledges the release', async () => { const ctx = context() await evictStructuredAgentSession(ctx) expect(ctx.order).toEqual([ @@ -62,7 +62,7 @@ describe('structured agent session eviction', () => { 'close', 'discardSink', 'releaseLease', - 'forget' + 'acknowledgeRelease' ]) }) @@ -90,7 +90,7 @@ describe('structured agent session eviction', () => { 'close-sink', 'discard-sink', 'release-lease', - 'forget-session' + 'acknowledge-release' ]) }) @@ -114,7 +114,7 @@ describe('structured agent session eviction', () => { } }) - it('aborts after a failed drain barrier without unbinding or forgetting the session', async () => { + it('aborts after a failed drain barrier without unbinding or acknowledging the release', async () => { const ctx = context() ctx.eventSink.drained = vi.fn(async () => { ctx.order.push('drained') @@ -128,7 +128,7 @@ describe('structured agent session eviction', () => { expect(ctx.eventSink.close).not.toHaveBeenCalled() expect(ctx.discardSink).not.toHaveBeenCalled() expect(ctx.releaseLease).not.toHaveBeenCalled() - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() expect(ctx.order).toEqual(['closeSession', 'drained']) }) }) @@ -154,9 +154,9 @@ describe('rows the provider emits while closing', () => { return true } } as never, - forget: async () => {}, discardSink: () => state.discardEventSink(sessionId), - releaseLease: async () => {} + releaseLease: async () => {}, + acknowledgeRelease: () => {} }) expect(published).toEqual(['final-flush']) @@ -174,9 +174,13 @@ describe('a child that will not stop', () => { ctx.adapter.closeSession = vi.fn(async () => { throw error }) + const stopped = vi.fn() + ctx.onProviderChildStopped = stopped await evictStructuredAgentSession(ctx) + // The host ends its child on the one reading of the verdict, not a second one of its own. + expect(stopped).toHaveBeenCalledWith({ rootGone: true }) expect(ctx.order).toEqual([ 'drained', 'settleWork', @@ -184,18 +188,18 @@ describe('a child that will not stop', () => { 'close', 'discardSink', 'releaseLease', - 'forget' + 'acknowledgeRelease' ]) }) - it('aborts without forgetting the session, so the next close is a real retry', async () => { + it('aborts without acknowledging the release, so the next stop is a real retry', async () => { const ctx = context() ctx.adapter.closeSession = vi.fn(async () => false) await expect(evictStructuredAgentSession(ctx)).rejects.toMatchObject({ step: 'stop-provider-child' }) - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() expect(ctx.discardSink).not.toHaveBeenCalled() expect(ctx.order).toEqual([]) }) @@ -210,7 +214,7 @@ describe('a child that will not stop', () => { StructuredAgentSessionEvictionError ) expect(ctx.eventSink.close).not.toHaveBeenCalled() - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() }) }) @@ -225,9 +229,9 @@ describe('eviction against the real sink cache', () => { sessionId, eventSink: state.eventSinkFor(sessionId), adapter: { closeSession: async () => true } as never, - forget: async () => {}, discardSink: () => state.discardEventSink(sessionId), - releaseLease: async () => {} + releaseLease: async () => {}, + acknowledgeRelease: () => {} }) const published: string[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts index e5a1d942818c..2c03143ff417 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts @@ -1,4 +1,4 @@ -// Releasing one structured session's resources. +// Stopping one structured session's provider child and handing its lease back. // // Teardown is a DATA list, not a method body, for the reason this file exists at all: the host // tracked which sessions were live in a map, and tore them down at three unrelated call sites @@ -22,6 +22,7 @@ import { type StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionStopVerdict } from './structured-agent-session-host-types' import { withTimeout } from '../../../shared/promise-timeout-fallback' export type StructuredAgentSessionEvictionContext = { @@ -29,17 +30,17 @@ export type StructuredAgentSessionEvictionContext = { hasProviderChild?: boolean eventSink: DeferredStructuredAgentSessionEventSink adapter: StructuredAgentSessionAdapter - /** Closes the session's journal handle and drops the map entry. Async and - * awaited: `close()` is ordered behind queued writes, and a delete that - * returns while the close is still queued leaves nothing to retry. */ - forget: () => Promise + /** Tells the adapter the released lease is done with, so it drops this child's route and index. + * The conversation stays: stopping the agent never closes its journal. */ + acknowledgeRelease: () => Promise | void /** Drops the cached sink so a later attach mints a fresh one. */ discardSink: () => void /** Fires right before the stop, while the child's turn and background roster are still live. A * throw is logged, never allowed to abort the stop. */ beforeProviderChildStop?: () => void - /** Fires once the adapter has PROVEN the child gone, so host bookkeeping stops claiming one. */ - onProviderChildStopped?: () => void + /** Fires with the stop's verdict once `stopAgentSessionProviderRoot` read the root gone, so host + * bookkeeping stops claiming a child. */ + onProviderChildStopped?: (verdict: StructuredAgentSessionStopVerdict) => void /** Whether this host still owes the child's wind-down. Distinct from `hasProviderChild`, which a * proven exit retires mid-run: the two disagree for exactly the steps a retry has to repeat. */ owesProviderChildWindDown?: boolean @@ -84,13 +85,13 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe } // An adapter with no close has nothing to stop; anything else must PROVE the exit. const stop = context.adapter.disposeSession ?? context.adapter.closeSession - if ( - stop && - !(await stopAgentSessionProviderRoot(() => stop.call(context.adapter, context.sessionId))) - ) { + const rootGone = stop + ? await stopAgentSessionProviderRoot(() => stop.call(context.adapter, context.sessionId)) + : true + if (!rootGone) { throw new Error('provider child exit was not proven') } - context.onProviderChildStopped?.() + context.onProviderChildStopped?.({ rootGone }) } }, { @@ -115,11 +116,11 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe // these two; eviction has to as well. { name: 'discard-sink', run: (context) => context.discardSink() }, // Why here and not last: the durable lease still names a process this host just stopped, and a - // record left claiming a live owner is one nothing can resume — the next surface to open the - // chat would find a session it may not acquire. Placed BEFORE forget so a release that cannot - // be written aborts while the session is still indexed, which is what makes the retry real. + // record left claiming a live owner is one nothing can resume — the next send would find a + // session it may not acquire. Placed BEFORE the acknowledgement so a release that cannot be + // written aborts while the adapter still routes the session, which is what makes the retry real. { name: 'release-lease', run: (context) => context.releaseLease() }, - { name: 'forget-session', run: (context) => context.forget() } + { name: 'acknowledge-release', run: (context) => context.acknowledgeRelease() } ] export class StructuredAgentSessionEvictionError extends Error { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts index 20531bd64653..90e07ddf88ca 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts @@ -21,11 +21,10 @@ import type { StructuredAgentSessionAttachContext } from './structured-agent-ses import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' -// Everything before the journal is out of scope here; what matters is that the orchestration's -// own `onAttachFailed` runs, which is the real one. +// Everything before the journal is out of scope here; what matters is what the orchestration does +// when the attach throws after acquisition. vi.mock('./structured-agent-session-attach-flow', () => ({ - performAttach: async (input: { onAttachFailed?: () => Promise }) => { - await input.onAttachFailed?.() + performAttach: async () => { throw new Error('attach failed after acquisition') } })) @@ -136,10 +135,7 @@ async function workingSession(): Promise<{ accountHome: ownerRecord().accountHome, runtimeKind: 'native' }, - fence: 1, - hasProviderChild: true, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: { generation: null, fence: 1, phase: 'ready' } } ] ]) @@ -190,7 +186,7 @@ function attachContext( reconcileLeases: async () => null, serialize: (_sessionId: string, task: () => Promise) => task(), now: () => 1, - forgetStatus: (sessionId: string) => feed.forget(sessionId) + publishStatus: (sessionId: string) => feed.publish(sessionId) } as unknown as StructuredAgentSessionAttachContext } @@ -230,15 +226,19 @@ describe('a session that leaves the host without an explicit close', () => { expect(server.getStatusSnapshot()).toEqual([expect.objectContaining({ prompt: 'other host' })]) }) - it('leaves the agent-status store with it when an attach fails', async () => { + // A failed attach no longer drops the session: the conversation stays open for the failure to be + // written into, so its row stays with it and the later close forgets both together. + it('keeps the session and its status row together when an attach fails', async () => { const { server, feed, sessions } = await workingSession() + const drop = vi.spyOn(server, 'dropStructuredStatus') await expect( attachStructuredAgentSession(attachContext(sessions, feed), 'caller-1', attachParams) ).rejects.toThrow('attach failed after acquisition') - expect(sessions.has(SESSION)).toBe(false) - expect(server.getStatusSnapshot()).toEqual([]) + expect(sessions.has(SESSION)).toBe(true) + expect(drop).not.toHaveBeenCalled() + expect(server.getStatusSnapshot()).toHaveLength(1) }) // The feed's own cache deliberately retains the projection for reload history; only the store diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-grouped-prompt.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-grouped-prompt.test.ts index 013f4191c30a..8181110a5a6b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-grouped-prompt.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-grouped-prompt.test.ts @@ -96,7 +96,7 @@ async function seedGroupedQuestion(): Promise<{ itemId: string; revision: number }) await host.flushStreamedEvents(SESSION) const itemId = agentJournalItemKey(identity) - const page = host.history({ sessionId: SESSION, direction: 'tail' }) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) const appended = page.ok ? page.page.items.find((item) => item.itemId === itemId) : null if (!appended) { throw new Error('provider question was not written to the journal') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts deleted file mode 100644 index 974a69b1c3c8..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { runKeyedSerializedOperation } from '../../cli/keyed-promise-queue' -import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' -import { StructuredAgentSessionHolds } from './structured-agent-session-holds' - -const GRACE_MS = 15_000 -const pendingHolds: StructuredAgentSessionHolds[] = [] - -/** The host's per-session queue: a second hold waits for the attach the first one is running. */ -function keyedSerialize() { - const chains = new Map>() - return (sessionId: string, task: () => Promise) => - runKeyedSerializedOperation(chains, sessionId, task) -} - -function resumeHarness() { - const resumeGate = Promise.withResolvers() - let child = false - let turnActive = false - const evict = vi.fn(async () => { - child = false - }) - const resume = vi.fn(async () => { - await resumeGate.promise - child = true - return { ok: true as const } - }) - const holds = new StructuredAgentSessionHolds({ - resume, - serialize: keyedSerialize(), - hasProviderChild: () => child, - hasOwedWork: () => turnActive, - evict, - graceMs: GRACE_MS - }) - pendingHolds.push(holds) - return { - holds, - resume, - resumeGate, - evict, - hasChild: () => child, - setTurnActive: (value: boolean) => { - turnActive = value - } - } -} - -beforeEach(() => vi.useFakeTimers()) -afterEach(() => { - for (const holds of pendingHolds.splice(0)) { - holds.dispose() - } - vi.useRealTimers() -}) - -describe('a surface leaving while its structured session resumes', () => { - it('releases the acquired child after the last surface disconnects during resume', async () => { - const { holds, resumeGate, evict, hasChild } = resumeHarness() - const hold = holds.hold('session-1', 'connection-1:chat') - - holds.release('session-1', 'connection-1:chat') - expect(holds.isReleasePending('session-1')).toBe(false) - resumeGate.resolve() - await hold - - expect(hasChild()).toBe(true) - expect(holds.isHeld('session-1')).toBe(false) - expect(holds.isReleasePending('session-1')).toBe(true) - await vi.advanceTimersByTimeAsync(GRACE_MS - 1) - expect(evict).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - expect(evict).toHaveBeenCalledExactlyOnceWith('session-1') - expect(hasChild()).toBe(false) - }) - - it('waits for an active turn before releasing the late child', async () => { - const { holds, resumeGate, evict, setTurnActive } = resumeHarness() - const hold = holds.hold('session-1', 'connection-1:chat') - holds.release('session-1', 'connection-1:chat') - setTurnActive(true) - resumeGate.resolve() - await hold - - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(evict).not.toHaveBeenCalled() - expect(holds.isReleasePending('session-1')).toBe(true) - - setTurnActive(false) - await vi.advanceTimersByTimeAsync(GRACE_MS) - expect(evict).toHaveBeenCalledExactlyOnceWith('session-1') - }) - - it.each([false, true])('preserves an arriving holder with resume=%s', async (resume) => { - const { holds, resumeGate, evict } = resumeHarness() - const first = holds.hold('session-1', 'connection-1:chat') - holds.release('session-1', 'connection-1:chat') - const replacement = holds.hold('session-1', 'connection-2:chat', { resume }) - resumeGate.resolve() - await Promise.all([first, replacement]) - - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(holds.isHeld('session-1')).toBe(true) - expect(holds.isReleasePending('session-1')).toBe(false) - expect(evict).not.toHaveBeenCalled() - - holds.release('session-1', 'connection-2:chat') - await vi.advanceTimersByTimeAsync(GRACE_MS) - expect(evict).toHaveBeenCalledExactlyOnceWith('session-1') - }) - - it('cancels the late-child release when a surface reconnects during grace', async () => { - const { holds, resumeGate, evict } = resumeHarness() - const hold = holds.hold('session-1', 'connection-1:chat') - holds.release('session-1', 'connection-1:chat') - resumeGate.resolve() - await hold - expect(holds.isReleasePending('session-1')).toBe(true) - - await holds.hold('session-1', 'connection-2:chat') - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(holds.isReleasePending('session-1')).toBe(false) - expect(evict).not.toHaveBeenCalled() - }) - - it('preserves a failed resume without scheduling eviction', async () => { - const { holds, resumeGate, evict, hasChild } = resumeHarness() - const failure = new Error('provider acquisition failed') - const hold = holds.hold('session-1', 'connection-1:chat') - const rejected = expect(hold).rejects.toBe(failure) - holds.release('session-1', 'connection-1:chat') - resumeGate.reject(failure) - await rejected - - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(hasChild()).toBe(false) - expect(holds.isHeld('session-1')).toBe(false) - expect(holds.isReleasePending('session-1')).toBe(false) - expect(evict).not.toHaveBeenCalled() - }) - - it('leaves late acquisition cleanup to host teardown after disposal', async () => { - const { holds, resumeGate, evict } = resumeHarness() - const hold = holds.hold('session-1', 'connection-1:chat') - holds.release('session-1', 'connection-1:chat') - holds.dispose() - resumeGate.resolve() - await hold - - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(holds.isReleasePending('session-1')).toBe(false) - expect(evict).not.toHaveBeenCalled() - }) - - it('does not restart release timers when a surface leaves after disposal', async () => { - const { holds, resumeGate, evict } = resumeHarness() - const hold = holds.hold('session-1', 'connection-1:chat') - resumeGate.resolve() - await hold - holds.dispose() - holds.release('session-1', 'connection-1:chat') - - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(holds.isHeld('session-1')).toBe(false) - expect(holds.isReleasePending('session-1')).toBe(false) - expect(evict).not.toHaveBeenCalled() - }) - - it('releases a late child acquired after explicit close forgot its holders', async () => { - const { holds, resumeGate, evict } = resumeHarness() - const hold = holds.hold('session-1', 'connection-1:chat') - holds.forget('session-1') - resumeGate.resolve() - await hold - - await vi.advanceTimersByTimeAsync(GRACE_MS) - expect(holds.isHeld('session-1')).toBe(false) - expect(evict).toHaveBeenCalledExactlyOnceWith('session-1') - }) - - it('lets a holder that left and came back make its own attempt behind a failing one, and its failure releases it', async () => { - const { holds, resume, resumeGate, evict } = resumeHarness() - const first = holds.hold('session-1', 'same-holder') - const firstRejected = expect(first).rejects.toThrow('acquisition failed') - holds.release('session-1', 'same-holder') - const replacement = holds.hold('session-1', 'same-holder') - const replacementRejected = expect(replacement).rejects.toThrow('acquisition failed') - await vi.advanceTimersByTimeAsync(0) - expect(holds.isHeld('session-1')).toBe(true) - expect(resume).toHaveBeenCalledOnce() - - // The gate stays rejected, so the replacement's own attempt fails the same way. - resumeGate.reject(new Error('acquisition failed')) - await Promise.all([firstRejected, replacementRejected]) - - expect(resume).toHaveBeenCalledTimes(2) - expect(holds.isHeld('session-1')).toBe(false) - expect(holds.isReleasePending('session-1')).toBe(false) - await vi.advanceTimersByTimeAsync(GRACE_MS * 2) - expect(evict).not.toHaveBeenCalled() - }) - - it('keeps a re-hold that finds the child the failing attempt ahead of it left behind', async () => { - const gate = Promise.withResolvers() - let child = false - const holds = new StructuredAgentSessionHolds({ - // The child is up before the attempt settles, and then the attempt fails behind it. - resume: async () => { - child = true - await gate.promise - return { ok: true as const } - }, - serialize: keyedSerialize(), - hasProviderChild: () => child, - hasOwedWork: () => false, - evict: async () => {}, - graceMs: GRACE_MS - }) - pendingHolds.push(holds) - const first = holds.hold('session-1', 'same-holder') - const rejected = expect(first).rejects.toThrow('acquisition failed') - holds.release('session-1', 'same-holder') - const replacement = holds.hold('session-1', 'same-holder') - - gate.reject(new Error('acquisition failed')) - await rejected - // The first hold's failure released only the holder it added, at the incarnation it added; - // the replacement then ran, found the child, and kept the holder it re-took. - await replacement - - expect(holds.isHeld('session-1')).toBe(true) - expect(holds.isReleasePending('session-1')).toBe(false) - }) - - it('starts a fresh resume once the failed one has settled', async () => { - let child = false - const resume = vi - .fn<() => Promise>() - .mockRejectedValueOnce(new Error('first acquisition failed')) - .mockImplementationOnce(async () => { - child = true - return { ok: true } - }) - const holds = new StructuredAgentSessionHolds({ - resume, - serialize: keyedSerialize(), - hasProviderChild: () => child, - hasOwedWork: () => false, - evict: async () => {}, - graceMs: GRACE_MS - }) - pendingHolds.push(holds) - - await expect(holds.hold('session-1', 'chat-1')).rejects.toThrow('first acquisition failed') - await holds.hold('session-1', 'chat-1') - - expect(resume).toHaveBeenCalledTimes(2) - expect(holds.isHeld('session-1')).toBe(true) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holders.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holders.ts deleted file mode 100644 index 531717168b17..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holders.ts +++ /dev/null @@ -1,71 +0,0 @@ -// Who WANTS this session alive, as a set of ids rather than a count. -// -// A refcount is the obvious shape and the wrong one. Every path that decrements it — a chat tab -// closing, a transport dying, a client retrying a release it already sent — can fire twice or not -// at all, and an integer cannot tell those apart: a duplicate release evicts a session somebody is -// still looking at, and a lost one leaks the child forever. A set answers both idempotently, -// because it records WHICH surface holds the session, not how many do. - -type Holder = { resumeCapable: boolean; incarnation: symbol } - -export class StructuredAgentSessionHolders { - private readonly bySession = new Map>() - - /** True when the session gained its FIRST holder — the edge that ends a pending release. */ - add(sessionId: string, holderId: string, resumeCapable = true): boolean { - const holders = this.bySession.get(sessionId) - if (!holders) { - this.bySession.set(sessionId, new Map([[holderId, { resumeCapable, incarnation: Symbol() }]])) - return true - } - const previous = holders.get(holderId) - holders.set(holderId, { - resumeCapable: (previous?.resumeCapable ?? false) || resumeCapable, - incarnation: previous?.incarnation ?? Symbol() - }) - return false - } - - /** True when the session lost its LAST holder — the edge that starts one. */ - remove(sessionId: string, holderId: string, expectedIncarnation?: symbol): boolean { - const holders = this.bySession.get(sessionId) - if ( - expectedIncarnation !== undefined && - holders?.get(holderId)?.incarnation !== expectedIncarnation - ) { - return false - } - if (!holders?.delete(holderId) || holders.size > 0) { - return false - } - this.bySession.delete(sessionId) - return true - } - - isHeld(sessionId: string): boolean { - return (this.bySession.get(sessionId)?.size ?? 0) > 0 - } - - has(sessionId: string, holderId: string): boolean { - return this.bySession.get(sessionId)?.has(holderId) ?? false - } - - incarnation(sessionId: string, holderId: string): symbol | undefined { - return this.bySession.get(sessionId)?.get(holderId)?.incarnation - } - - holderIds(sessionId: string): string[] { - return [...(this.bySession.get(sessionId)?.keys() ?? [])] - } - - hasResumeCapableHolder(sessionId: string): boolean { - return [...(this.bySession.get(sessionId)?.values() ?? [])].some( - (holder) => holder.resumeCapable - ) - } - - /** Drops every holder of one session without evaluating the edge, for a session that is gone. */ - forget(sessionId: string): void { - this.bySession.delete(sessionId) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts deleted file mode 100644 index 68027f81887c..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts +++ /dev/null @@ -1,346 +0,0 @@ -// The parts a session's lifetime is assembled from: the holder set, the release clock, and the -// deadline that keeps teardown from hanging. - -import { afterEach, describe, expect, it, vi } from 'vitest' -import { runKeyedSerializedOperation } from '../../cli/keyed-promise-queue' -import { StructuredAgentSessionHolders } from './structured-agent-session-holders' -import { StructuredAgentSessionReleaseClock } from './structured-agent-session-release-clock' -import { StructuredAgentSessionHolds } from './structured-agent-session-holds' -import { - STRUCTURED_AGENT_SESSION_EVICTION_STEPS, - evictStructuredAgentSession -} from './structured-agent-session-eviction' -import { - StructuredAgentSessionEvictionTimeoutError, - withStructuredAgentSessionEvictionDeadline -} from './structured-agent-session-eviction-deadline' - -const clocks: StructuredAgentSessionReleaseClock[] = [] - -/** The host's per-session queue, so a hold and a writer really take turns. */ -function keyedSerialize() { - const chains = new Map>() - return (sessionId: string, task: () => Promise) => - runKeyedSerializedOperation(chains, sessionId, task) -} - -function clock(deps: { - hasOwedWork?: () => boolean - isHeld?: () => boolean - evict: (sessionId: string) => Promise - onError?: (input: { sessionId: string; error: unknown }) => void -}): StructuredAgentSessionReleaseClock { - const created = new StructuredAgentSessionReleaseClock({ - hasOwedWork: deps.hasOwedWork ?? (() => false), - isHeld: deps.isHeld ?? (() => false), - evict: deps.evict, - ...(deps.onError ? { onError: deps.onError } : {}), - graceMs: 1 - }) - clocks.push(created) - return created -} - -afterEach(() => { - for (const created of clocks.splice(0)) { - created.dispose() - } -}) - -describe('the holder set', () => { - it('reports the first and last holder, and nothing in between', () => { - const holders = new StructuredAgentSessionHolders() - - expect(holders.add('session-1', 'a')).toBe(true) - expect(holders.add('session-1', 'b')).toBe(false) - expect(holders.remove('session-1', 'a')).toBe(false) - expect(holders.remove('session-1', 'b')).toBe(true) - expect(holders.isHeld('session-1')).toBe(false) - }) - - // The reason this is a set and not a count: every release path can fire twice or not at all. - it('absorbs a duplicate hold and a duplicate release', () => { - const holders = new StructuredAgentSessionHolders() - - holders.add('session-1', 'a') - holders.add('session-1', 'a') - expect(holders.remove('session-1', 'a')).toBe(true) - expect(holders.remove('session-1', 'a')).toBe(false) - expect(holders.holderIds('session-1')).toEqual([]) - }) - - it('keeps one session holders out of another session holders', () => { - const holders = new StructuredAgentSessionHolders() - - holders.add('session-1', 'a') - holders.add('session-2', 'a') - holders.remove('session-1', 'a') - - expect(holders.isHeld('session-1')).toBe(false) - expect(holders.isHeld('session-2')).toBe(true) - }) - - it('distinguishes retaining holders from holders that may resume a provider', () => { - const holders = new StructuredAgentSessionHolders() - - holders.add('session-1', 'subscriber', false) - expect(holders.hasResumeCapableHolder('session-1')).toBe(false) - - holders.add('session-1', 'chat', true) - expect(holders.hasResumeCapableHolder('session-1')).toBe(true) - holders.remove('session-1', 'chat') - expect(holders.hasResumeCapableHolder('session-1')).toBe(false) - }) -}) - -describe('the release clock', () => { - it('waits out a running turn instead of evicting into it', async () => { - const evict = vi.fn(async () => {}) - let turnRunning = true - const releasing = clock({ hasOwedWork: () => turnRunning, evict }) - - releasing.arm('session-1') - await new Promise((resolve) => setTimeout(resolve, 30)) - expect(evict).not.toHaveBeenCalled() - - turnRunning = false - await vi.waitFor(() => expect(evict).toHaveBeenCalledWith('session-1')) - }) - - it('stands down when a holder arrives during the wait', async () => { - const evict = vi.fn(async () => {}) - const releasing = clock({ isHeld: () => true, evict }) - - releasing.arm('session-1') - await new Promise((resolve) => setTimeout(resolve, 30)) - - expect(evict).not.toHaveBeenCalled() - }) - - it('keeps an idle unheld child for thirty minutes, and activity starts the window over', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - try { - const evict = vi.fn(async () => {}) - const idle = new StructuredAgentSessionReleaseClock({ - hasOwedWork: () => false, - isHeld: () => false, - evict - }) - clocks.push(idle) - const minutes = (count: number) => vi.advanceTimersByTimeAsync(count * 60_000) - - idle.arm('session-1') - await minutes(20) - idle.renew('session-1') - await minutes(29) - expect(evict).not.toHaveBeenCalled() - - await minutes(1) - expect(evict).toHaveBeenCalledWith('session-1') - } finally { - vi.useRealTimers() - } - }) - - it('does not start a window for a session nothing released', async () => { - const evict = vi.fn(async () => {}) - const releasing = clock({ evict }) - - releasing.renew('session-1') - await new Promise((resolve) => setTimeout(resolve, 30)) - - expect(releasing.isArmed('session-1')).toBe(false) - expect(evict).not.toHaveBeenCalled() - }) - - it('reports a failed eviction rather than swallowing it', async () => { - const onError = vi.fn() - const releasing = clock({ - evict: async () => { - throw new Error('child would not stop') - }, - onError - }) - - releasing.arm('session-1') - - await vi.waitFor(() => - expect(onError).toHaveBeenCalledWith({ - sessionId: 'session-1', - error: expect.objectContaining({ message: 'child would not stop' }) - }) - ) - }) -}) - -describe('holds', () => { - it('resumes a session on its first hold and not on a retained one', async () => { - let child = false - const resume = vi.fn(async () => { - child = true - return { ok: true as const } - }) - const holds = new StructuredAgentSessionHolds({ - resume, - serialize: keyedSerialize(), - hasProviderChild: () => child, - hasOwedWork: () => false, - evict: async () => {}, - graceMs: 1 - }) - - await holds.hold('session-1', 'stream-1', { resume: false }) - expect(resume).not.toHaveBeenCalled() - - await holds.hold('session-1', 'chat-1') - expect(resume).toHaveBeenCalledOnce() - - child = true - await holds.hold('session-1', 'chat-2') - expect(resume).toHaveBeenCalledOnce() - holds.dispose() - }) - - it('runs one resume for a writer and a hold that ask in the same gap', async () => { - const gate = Promise.withResolvers() - let child = false - const resume = vi.fn(async () => { - await gate.promise - child = true - return { ok: true as const } - }) - const serialize = keyedSerialize() - const holds = new StructuredAgentSessionHolds({ - resume, - serialize, - hasProviderChild: () => child, - hasOwedWork: () => false, - evict: async () => {}, - graceMs: 1 - }) - - // A send's ensure-owner step: already inside the session's serialize when it asks. - const writer = serialize('session-1', () => holds.ensureProviderChild('session-1')) - const hold = holds.hold('session-1', 'chat-1') - gate.resolve() - - await expect(writer).resolves.toEqual({ ok: true }) - await hold - // The hold ran after the writer's step and found the child: nothing to resume. - expect(resume).toHaveBeenCalledOnce() - // The surface arrived while the writer's resume ran, so the child it got is held, not idle. - expect(holds.isHeld('session-1')).toBe(true) - expect(holds.isReleasePending('session-1')).toBe(false) - holds.dispose() - }) - - it('puts a child a writer resumed with no surface on the idle clock', async () => { - let child = false - const serialize = keyedSerialize() - const holds = new StructuredAgentSessionHolds({ - resume: async () => { - child = true - return { ok: true as const } - }, - serialize, - hasProviderChild: () => child, - hasOwedWork: () => false, - evict: async () => {}, - graceMs: 60_000 - }) - - await serialize('session-1', () => holds.ensureProviderChild('session-1')) - - expect(holds.isHeld('session-1')).toBe(false) - expect(holds.isReleasePending('session-1')).toBe(true) - holds.dispose() - }) - - it('never arms the clock for a session with nothing to stop', async () => { - const evict = vi.fn(async () => {}) - const holds = new StructuredAgentSessionHolds({ - resume: async () => ({ ok: true as const }), - serialize: keyedSerialize(), - hasProviderChild: () => false, - hasOwedWork: () => false, - evict, - graceMs: 1 - }) - - await holds.hold('session-1', 'chat-1', { resume: false }) - holds.release('session-1', 'chat-1') - await new Promise((resolve) => setTimeout(resolve, 20)) - - expect(evict).not.toHaveBeenCalled() - expect(holds.isReleasePending('session-1')).toBe(false) - holds.dispose() - }) - - it('fails a write-capable hold when resume proves no provider child', async () => { - const holds = new StructuredAgentSessionHolds({ - resume: async () => ({ ok: true as const }), - serialize: keyedSerialize(), - hasProviderChild: () => false, - hasOwedWork: () => false, - evict: async () => {}, - graceMs: 1 - }) - - await expect(holds.hold('session-1', 'chat-1')).rejects.toThrow( - 'agent_session_ownership_unknown' - ) - expect(holds.isHeld('session-1')).toBe(false) - holds.dispose() - }) -}) - -describe('the teardown deadline', () => { - it('leaves the child loaded instead of forcing it, and keeps the session indexed', async () => { - const forget = vi.fn() - const releaseLease = vi.fn(async () => {}) - - await expect( - evictStructuredAgentSession( - { - sessionId: 'session-1', - eventSink: { - unbind: vi.fn(), - drained: vi.fn(async () => {}), - close: vi.fn() - } as never, - adapter: { closeSession: () => new Promise(() => {}) } as never, - forget, - discardSink: vi.fn(), - releaseLease - }, - withStructuredAgentSessionEvictionDeadline(STRUCTURED_AGENT_SESSION_EVICTION_STEPS, 5) - ) - ).rejects.toMatchObject({ step: 'stop-provider-child' }) - - expect(forget).not.toHaveBeenCalled() - expect(releaseLease).not.toHaveBeenCalled() - }) - - it('names the step that ran out of time', async () => { - const [step] = withStructuredAgentSessionEvictionDeadline( - [{ name: 'slow-step', run: () => new Promise(() => {}) }], - 5 - ) - - await expect(step?.run({} as never)).rejects.toBeInstanceOf( - StructuredAgentSessionEvictionTimeoutError - ) - }) - - it('does not delay a step that finishes', async () => { - const ran: string[] = [] - const steps = withStructuredAgentSessionEvictionDeadline( - [{ name: 'fast-step', run: () => void ran.push('fast-step') }], - 5_000 - ) - - await steps[0]?.run({} as never) - - expect(ran).toEqual(['fast-step']) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts deleted file mode 100644 index 0acbccb6b55f..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts +++ /dev/null @@ -1,179 +0,0 @@ -// The lifetime of a structured session, tied to the surfaces that want one. -// -// Nothing used to tell the host that a chat WANTED a session, and nothing told it when a chat -// stopped wanting one. Both halves of that gap cost real processes: sessions nobody had opened got -// an app-server at every launch, and sessions the user closed kept theirs until the app quit. -// -// A surface takes a hold when it binds and drops it when it goes away. The first hold on a session -// with no child resumes it — that, and not the shape of a lease on disk, is what makes a provider -// process exist. The last hold leaving starts the idle release clock. Transport close is the BACKSTOP, -// not the mechanism: a client that vanishes mid-flight never sends its release, so the caller -// registers one against the connection and the holder set absorbs the duplicate. -// -// A send to a childless session resumes it too, and so does provider-exit recovery under an open -// surface. All three go through `ensureProviderChild` inside the session's serialize, so they take -// turns: the first to run attaches, and the next finds the child and attaches nothing. Two attaches -// for one session would race against the same released fence, and the loser's stale fence refused -// it — a hold that lost dropped its holder, a send that lost was refused. - -import { - StructuredAgentSessionReleaseClock, - type StructuredAgentSessionReleaseClockDeps -} from './structured-agent-session-release-clock' -import { StructuredAgentSessionHolders } from './structured-agent-session-holders' -import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' -import type { StructuredAgentSessionAttachOptions } from './structured-agent-session-attach-orchestration' - -export type StructuredAgentSessionHoldsDeps = { - /** Attaches a provider child, for a caller already inside `serialize`. */ - resume: ( - sessionId: string, - attachOptions?: StructuredAgentSessionAttachOptions - ) => Promise - serialize: (sessionId: string, task: () => Promise) => Promise - /** Whether evicting this session would actually free anything. */ - hasProviderChild: (sessionId: string) => boolean - hasOwedWork: (sessionId: string) => boolean - evict: (sessionId: string) => Promise - onError?: (input: { sessionId: string; error: unknown }) => void - graceMs?: number -} - -export type StructuredAgentSessionHoldOptions = { - /** False for a hold that only RETAINS — a subscription stream, which must not make a child - * exist just by reading history. */ - resume?: boolean -} - -export class StructuredAgentSessionHolds { - private readonly holders = new StructuredAgentSessionHolders() - private readonly clock: StructuredAgentSessionReleaseClock - private disposed = false - - constructor(private readonly deps: StructuredAgentSessionHoldsDeps) { - const clockDeps: StructuredAgentSessionReleaseClockDeps = { - hasOwedWork: deps.hasOwedWork, - isHeld: (sessionId) => this.holders.isHeld(sessionId), - evict: (sessionId) => this.deps.evict(sessionId), - ...(deps.onError ? { onError: deps.onError } : {}), - ...(deps.graceMs === undefined ? {} : { graceMs: deps.graceMs }) - } - this.clock = new StructuredAgentSessionReleaseClock(clockDeps) - } - - async hold( - sessionId: string, - holderId: string, - options: StructuredAgentSessionHoldOptions = {} - ): Promise { - const alreadyHeld = this.holders.has(sessionId, holderId) - this.holders.add(sessionId, holderId, options.resume !== false) - const incarnation = this.holders.incarnation(sessionId, holderId) - // Unconditional, not only on the first-holder edge: a second surface arriving during the grace - // window must cancel the pending release too. - this.clock.cancel(sessionId) - if (options.resume === false) { - return - } - let resumed: StructuredAgentSessionResumeOutcome - try { - resumed = await this.deps.serialize(sessionId, () => this.ensureProviderChild(sessionId)) - } catch (error) { - this.releaseFailedHold(sessionId, holderId, alreadyHeld, incarnation) - throw error - } - if (!resumed.ok) { - this.releaseFailedHold(sessionId, holderId, alreadyHeld, incarnation) - // The RPC surface raises a refusal as its code. - throw new Error(resumed.refusal.code) - } - } - - /** Only the holder this call added, at the incarnation it added: a same-ID hold that left and - * came back while this one waited owns the holder now, and its own attempt decides it. */ - private releaseFailedHold( - sessionId: string, - holderId: string, - alreadyHeld: boolean, - incarnation: symbol | undefined - ): void { - if (!alreadyHeld && incarnation !== undefined) { - this.release(sessionId, holderId, incarnation) - } - } - - /** - * Gives the session a provider child if it has none. - * - * For a caller already inside the session's serialize, which is what makes "if it has none" - * exact: a hold and a send that both find the owner gone run this in turn, and the second sees - * the first one's child. Each caller makes at most one attach, and a failed one leaves the - * next caller to make its own. With no surface holding the session afterwards, the child goes - * on the same clock a departed surface would start — including the surface that held it when - * provider-exit recovery began and left while the attach ran. - */ - async ensureProviderChild( - sessionId: string, - attachOptions?: StructuredAgentSessionAttachOptions - ): Promise { - if (this.deps.hasProviderChild(sessionId)) { - return { ok: true } - } - const resumed = await this.deps.resume(sessionId, attachOptions) - if (!resumed.ok) { - return resumed - } - if (!this.deps.hasProviderChild(sessionId)) { - return { - ok: false, - refusal: { - code: 'agent_session_ownership_unknown', - message: 'The session attached without a provider child to write to.' - } - } - } - // The last surface can disconnect before acquisition makes a child available to release. - if (!this.disposed && !this.holders.isHeld(sessionId)) { - this.clock.arm(sessionId) - } - return { ok: true } - } - - /** Activity — a journal write, or a start reaching the work it held; only a pending release - * notices, and it restarts its full window. */ - renew(sessionId: string): void { - this.clock.renew(sessionId) - } - - release(sessionId: string, holderId: string, expectedIncarnation?: symbol): void { - if (!this.holders.remove(sessionId, holderId, expectedIncarnation)) { - return - } - if (!this.disposed && this.deps.hasProviderChild(sessionId)) { - this.clock.arm(sessionId) - } - } - - /** Drops the holders of a session that is gone, whoever evicted it. */ - forget(sessionId: string): void { - this.clock.cancel(sessionId) - this.holders.forget(sessionId) - } - - isHeld(sessionId: string): boolean { - return this.holders.isHeld(sessionId) - } - - hasResumeCapableHolder(sessionId: string): boolean { - return this.holders.hasResumeCapableHolder(sessionId) - } - - isReleasePending(sessionId: string): boolean { - return this.clock.isArmed(sessionId) - } - - dispose(): void { - this.disposed = true - this.clock.dispose() - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts new file mode 100644 index 000000000000..46f20b9e67c3 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -0,0 +1,101 @@ +// The host's conversations: how one becomes open, and the delivery loop that hands its accepted +// messages to a provider child. Bundled because they share one invariant — a conversation open +// with a message queued has a delivery loop — and the open is where a loop for leftovers wakes. + +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + openStructuredAgentSessionConversation, + type OpenedStructuredAgentSessionConversation +} from './structured-agent-session-conversation-open' +import { StructuredAgentSessionDeliveryLoop } from './structured-agent-session-delivery-loop' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-agent-start' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' +import { structuredAgentSessionStartFailureText } from './structured-agent-session-send-preparation' +import { settleInterruptedCompaction } from './structured-compaction-recovery' +import { recoverStructuredRewind } from './structured-rewind-recovery' + +export type StructuredAgentSessionConversationDelivery = { + loop: StructuredAgentSessionDeliveryLoop + /** For a caller inside the session's serialize. */ + open: (sessionId: string) => Promise + /** Indexes a conversation some other open produced, as `open` would have. */ + adoptOpened: ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ) => Promise +} + +export function createStructuredAgentSessionConversationDelivery(input: { + deps: StructuredAgentSessionHostDeps + sessions: Map + serialize: (sessionId: string, task: () => Promise) => Promise + trackStart: (start: Promise) => Promise + /** Starts a child for `startedFor`, the queued message at the head, if the session has none. */ + ensureProviderChild: ( + sessionId: string, + startedFor: string + ) => Promise + reset: (sessionId: string, journal: AgentSessionJournal, reset: AgentJournalResetReason) => void + publishRestored: (sessionId: string) => void +}): StructuredAgentSessionConversationDelivery { + const { deps, sessions } = input + const loop = new StructuredAgentSessionDeliveryLoop({ + sessions, + adapter: deps.adapter, + serialize: input.serialize, + trackStart: input.trackStart, + ensureProviderChild: input.ensureProviderChild, + conversationFence: (sessionId) => + structuredAgentSessionConversationFence(deps.store, sessionId), + startFailureText: (sessionId, cause) => + structuredAgentSessionStartFailureText(deps.store.getRecord(sessionId), cause), + onError: (sessionId, error) => deps.onEventSinkError?.({ sessionId, error }) + }) + const adoptOpened = async ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ): Promise => { + const { session, reset } = opened + sessions.set(sessionId, session) + if (reset) { + input.reset(sessionId, session.journal, reset) + } + input.publishRestored(sessionId) + await settleInterruptedCommands(deps, sessionId, session) + if (session.journal.submissions().some(isQueuedAgentJournalSubmission)) { + loop.wake(sessionId) + } + } + return { + loop, + adoptOpened, + open: (sessionId) => + openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId) + } +} + +/** + * A compaction or rewind found prepared when the conversation opens was started under a child + * this process no longer has — the open runs only when none is indexed — so nothing will finish + * it, and left alone it refuses every send until a view attaches. Settled here instead of by a + * start inside acceptance. A Codex rewind only its provider can prove stays for the attach. + */ +async function settleInterruptedCommands( + deps: StructuredAgentSessionHostDeps, + sessionId: string, + session: StructuredAgentSessionHostSession +): Promise { + const fence = structuredAgentSessionConversationFence(deps.store, sessionId) + try { + await settleInterruptedCompaction(deps.store, sessionId, session.journal, fence) + await recoverStructuredRewind(deps.store, sessionId, session.journal, fence) + } catch (error) { + deps.onEventSinkError?.({ sessionId, error }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 0eed19031c7b..c2487161f605 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -1,27 +1,33 @@ -// The host's half of a session's lifetime: what a close does, and what a hold is wired to. +// The host's half of a session's lifetime: stopping its agent, and closing its conversation. // -// Lifted out of the host for the same reason attaching was — the host is a coordinator, and the -// sequence that stops a provider child and hands its lease back reads better next to the holder -// bookkeeping that decides when to run it than buried among the twenty other things a session can -// do. +// Two operations, because they end two different things. Stopping the agent ends the provider +// child and hands the lease back; the conversation — its open journal, its status row and its +// readers — stays, and the next send starts a new child. Closing the conversation drops the open +// journal handle, a cache the next read or write reopens. +// +// Both are written for a caller already inside the session's serialize: the queue is not +// reentrant, so every public entry point takes it once and calls these. -import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' -import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-projection' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { DISPATCH_REJECTED_PROVIDER_CLOSED } from '../../../shared/structured-agent-session-dispatch-rejection' import { evictStructuredAgentSession, STRUCTURED_AGENT_SESSION_EVICTION_STEPS, type StructuredAgentSessionEvictionContext } from './structured-agent-session-eviction' import { withStructuredAgentSessionEvictionDeadline } from './structured-agent-session-eviction-deadline' -import { StructuredAgentSessionHolds } from './structured-agent-session-holds' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { + StructuredAgentSessionChildEndCause, StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' +import { + endProviderChild, + structuredAgentSessionConversationFence +} from './structured-agent-session-provider-child' import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' -import { resumeHeldStructuredAgentSession } from './structured-agent-session-hold-resume' -import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' export type StructuredAgentSessionLifetimeContext = { @@ -29,8 +35,8 @@ export type StructuredAgentSessionLifetimeContext = { runtimeState: StructuredAgentSessionHostRuntimeState sessions: Map now: () => number - /** Drops the session's row from the agent-status store; see `forgetStructuredAgentSession`. */ - forgetStatus: (sessionId: string) => void + /** Re-projects the session's status after its agent stopped and the chat stays. */ + publishStatus?: (sessionId: string) => void /** Quit-only snapshot taken immediately before the provider child is stopped. */ restartWitness?: { beforeStop: (sessionId: string) => void @@ -38,73 +44,96 @@ export type StructuredAgentSessionLifetimeContext = { } } -/** Dropping a session and dropping its status row are ONE operation: the store keeps the row until - * told, so a caller that only deletes strands a live-looking row no reader can ever decay. */ -export async function forgetStructuredAgentSession( - context: StructuredAgentSessionLifetimeContext, - sessionId: string -): Promise { - await context.sessions.get(sessionId)?.journal.close() - context.sessions.delete(sessionId) - context.forgetStatus(sessionId) +type ConversationCloseDeps = Pick & { + store: Pick } -function hasProviderChild( - context: StructuredAgentSessionLifetimeContext, - sessionId: string -): boolean { - return context.sessions.get(sessionId)?.hasProviderChild === true +/** A conversation's handle closes with nothing queued: what is still queued when the chat closes, + * or the app quits, will not be handed over. Best effort: the next open's delivery loop rejects a + * leftover itself. */ +export async function abandonQueuedStructuredAgentSessionMessages( + deps: ConversationCloseDeps, + sessionId: string, + journal: StructuredAgentSessionHostSession['journal'] +): Promise { + await journal + .rejectQueuedSubmissions( + structuredAgentSessionConversationFence(deps.store, sessionId), + DISPATCH_REJECTED_PROVIDER_CLOSED + ) + .catch((error: unknown) => deps.onEventSinkError?.({ sessionId, error })) } /** The wind-down this host owes for the session's child. A live child always owes one, whatever a - * previous childless eviction recorded: a remembered `false` must never outrank the child in front - * of it. */ -function owesProviderChildWindDown(session: StructuredAgentSessionHostSession): boolean { - return session.hasProviderChild || session.owesProviderChildWindDown === true + * previous childless eviction recorded: a remembered tombstone must never outrank the child in + * front of it. */ +function owedProviderChildWindDown( + session: StructuredAgentSessionHostSession +): StructuredAgentSessionProviderChildIdentity | undefined { + return session.child + ? { generation: session.child.generation, fence: session.child.fence } + : session.owesProviderChildWindDown } -/** Runs the eviction steps under a deadline. A step that fails — or runs out of time — aborts the - * rest, which leaves the session indexed and the child loaded so the next close is a real retry. */ -export async function evictHeldStructuredAgentSession( +/** + * The agent goes to rest; the conversation stays. Runs the eviction steps under a deadline. A step + * that fails — or runs out of time — aborts the rest and leaves the wind-down owed, so the next + * stop is a real retry. `ending` is how the child's end is told: a user's Stop, the host stopping it + * for a cause (with its text), or an eviction the conversation's close follows. + */ +export async function stopStructuredAgentSessionAgentUnderSerialize( context: StructuredAgentSessionLifetimeContext, - sessionId: string + sessionId: string, + ending: { + cause: Extract + reason?: string + } = { cause: 'user-stop' } ): Promise { const session = context.sessions.get(sessionId) if (!session) { return } - // The obligation OUTLIVES the child. `hasProviderChild` is retired the instant the adapter - // proves the exit, so a step that aborts after that point would otherwise leave the retry - // reading "no child here" and skipping the settlement and the lease release it still owes. - const owesWindDown = owesProviderChildWindDown(session) - session.owesProviderChildWindDown = owesWindDown + // The obligation OUTLIVES the child. `child` is ended the instant the adapter proves the exit, + // so a step that aborts after that point would otherwise leave the retry reading "no child + // here" and skipping the settlement and the lease release it still owes. + const owed = owedProviderChildWindDown(session) + session.owesProviderChildWindDown = owed + const stopping = session.child let settlementError: unknown const eviction: StructuredAgentSessionEvictionContext = { sessionId, // The retry must not re-stop a child the adapter already proved gone, so this stays honest. - hasProviderChild: session.hasProviderChild, - owesProviderChildWindDown: owesWindDown, + hasProviderChild: stopping !== null, + owesProviderChildWindDown: owed !== undefined, eventSink: context.runtimeState.eventSinkFor(sessionId), adapter: context.deps.adapter, ...(context.restartWitness ? { beforeProviderChildStop: () => context.restartWitness?.beforeStop(sessionId) } : {}), - // Host state must not disagree with the adapter for the seven steps in between. - onProviderChildStopped: () => { - session.hasProviderChild = false + // Host state must not disagree with the adapter for the steps in between. + onProviderChildStopped: (verdict) => { + if (stopping) { + endProviderChild(session, { + generation: stopping.generation, + fence: stopping.fence, + cause: ending.cause, + reason: ending.reason ?? null, + duringStartup: stopping.phase === 'starting', + ...verdict + }) + } context.restartWitness?.stopped(sessionId) }, - forget: async () => { - await forgetStructuredAgentSession(context, sessionId) - context.deps.adapter.acknowledgeSessionRelease?.(sessionId) - }, + acknowledgeRelease: () => context.deps.adapter.acknowledgeSessionRelease?.(sessionId), discardSink: () => context.runtimeState.discardEventSink(sessionId), settleWork: async () => { + const fence = + owed?.fence ?? structuredAgentSessionConversationFence(context.deps.store, sessionId) const settled = await settleStructuredAgentSessionDeadGeneration({ journal: session.journal, sessionId, - fence: session.fence, - settlementId: `expected-close:${sessionId}:${session.fence}:${session.acquisitionGeneration ?? 'unknown'}`, + fence, + settlementId: `expected-close:${sessionId}:${fence}:${owed?.generation ?? 'unknown'}`, pendingSubmissionReason: 'provider_closed_before_acknowledgement', verdict: { state: 'interrupted', completedAt: context.now() }, showUnexpectedExitOutcome: false, @@ -114,20 +143,24 @@ export async function evictHeldStructuredAgentSession( } }) if (!settled) { - // Without the cause the quit log names the step and nothing else. + // Without the cause the log names the step and nothing else. throw new Error('dead generation work settlement failed', { cause: settlementError }) } }, releaseLease: async () => { - await releaseStoredStructuredAgentSessionOwner({ - store: context.deps.store, - sessionId, - hasProviderChild: owesWindDown, - expectedFence: session.fence, - now: context.now() - }) - session.owesProviderChildWindDown = false - context.forgetStatus(sessionId) + if (owed) { + await releaseStoredStructuredAgentSessionOwner({ + store: context.deps.store, + sessionId, + hasProviderChild: true, + expectedFence: owed.fence, + now: context.now() + }) + } + session.owesProviderChildWindDown = undefined + // Whatever ended the child, the row belongs to the conversation: it shows not-running, and + // only the conversation's close forgets it. + context.publishStatus?.(sessionId) } } await evictStructuredAgentSession( @@ -136,6 +169,41 @@ export async function evictHeldStructuredAgentSession( ) } +/** Whether the conversation's handle is only a cache now: no child, no wind-down owed, and nothing + * queued or waiting on the provider. */ +export function structuredAgentSessionConversationClosable( + session: StructuredAgentSessionHostSession +): boolean { + return ( + owedProviderChildWindDown(session) === undefined && + !session.journal.submissions().some(isQueuedAgentJournalSubmission) && + session.journal.pendingSubmissions().length === 0 + ) +} + +/** + * Drops the conversation's open handle. The entry leaves the map BEFORE the handle closes, so a + * lock-free reader sees an open handle or none — never one that is closing — and one arriving + * after the delete waits behind this step and reopens. Answers false, closing nothing, when the + * handle is still more than a cache. + */ +export async function closeStructuredAgentSessionConversationUnderSerialize( + context: Pick & { + /** The status row outlives the handle; see `StructuredAgentSessionClientDelivery`. */ + closeStatus: (sessionId: string) => void + }, + sessionId: string +): Promise { + const session = context.sessions.get(sessionId) + if (!session || !structuredAgentSessionConversationClosable(session)) { + return false + } + context.sessions.delete(sessionId) + context.closeStatus(sessionId) + await session.journal.close() + return true +} + /** Stops every provider child owned by this host while keeping failed evictions reachable. A * session whose child is already stopped but whose wind-down aborted is still in scope — that is * the retry. */ @@ -146,9 +214,9 @@ export async function evictOwnedStructuredAgentSessions( retainOnFailure: Set ): Promise { const ownedSessionIds = [...context.sessions] - .filter(([, session]) => owesProviderChildWindDown(session)) + .filter(([, session]) => owedProviderChildWindDown(session) !== undefined) .map(([sessionId]) => sessionId) - // Retained up front and cleared only once an eviction settles: the quit phase is bounded, and a + // Retained up front and cleared only once a stop settles: the quit phase is bounded, and a // timeout leaves these still running. Closing their journals underneath them is the one outcome // the retain set exists to prevent. for (const sessionId of ownedSessionIds) { @@ -159,7 +227,7 @@ export async function evictOwnedStructuredAgentSessions( ownedSessionIds.map(async (sessionId) => { try { await context.serialize(sessionId, () => - evictHeldStructuredAgentSession(context, sessionId) + stopStructuredAgentSessionAgentUnderSerialize(context, sessionId) ) retainOnFailure.delete(sessionId) } catch (error) { @@ -171,47 +239,3 @@ export async function evictOwnedStructuredAgentSessions( throw new AggregateError(failures, 'structured agent-session child eviction failed') } } - -/** The holds resume through the host's own attach, inside the session's serialize: a hold's - * resume and a send's ensure-owner step are the same serialized attach with a different asker. */ -export function createStructuredAgentSessionHolds( - attachContext: () => StructuredAgentSessionAttachContext, - close: (sessionId: string) => Promise -): StructuredAgentSessionHolds { - const context = attachContext() - return new StructuredAgentSessionHolds({ - resume: (sessionId, attachOptions) => - resumeHeldStructuredAgentSession({ - sessionId, - context: attachContext(), - callerKey: attachOptions?.admitRecoveryTicket - ? 'trusted-local:provider-exit-recovery' - : 'trusted-local:surface-hold', - ...(attachOptions ? { attachOptions } : {}) - }), - // Tracked from enqueue: a quit drains a queued resume before it evicts, so no child is - // spawned behind the eviction and orphaned. - serialize: (sessionId, task) => { - const current = attachContext() - return current.tasks.trackAttach(current.serialize(sessionId, task)) - }, - evict: close, - hasProviderChild: (sessionId) => hasProviderChild(context, sessionId), - // A send pending while the child is still starting is held for that start; evicting would - // refuse it. Any other pending send may wait on an echo that never comes, so eviction retires it. - // Subagents, commands and monitors outlive the lead's turn inside the child, so the live roster - // the sidebar shows as working is owed too; stopping the child would end them silently. - hasOwedWork: (sessionId) => { - const session = context.sessions.get(sessionId) - return session - ? activeStructuredAgentSessionTurnId(session.journal.snapshot().items) !== null || - (session.providerChildPhase === 'starting' && - session.journal.pendingSubmissions().length > 0) || - agentChildWorkLiveness(context.deps.adapter.backgroundTaskState?.(sessionId)?.tasks) !== - null - : false - }, - onError: (error) => context.deps.onEventSinkError?.(error), - ...(context.deps.releaseGraceMs === undefined ? {} : { graceMs: context.deps.releaseGraceMs }) - }) -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index c1ae42e237c6..0d19c7aacbe1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -1,10 +1,11 @@ -// Everything a client can ask an ATTACHED session to do: send a turn, cancel one, answer a prompt, -// change an option, read the options back. +// Everything a client can ask a session to do: send a turn, cancel one, answer a prompt, change an +// option, read the options back. // // They share one shape — admit the envelope against the lease, run a plan, publish the journal — so -// they share one path here rather than five copies in the host. The host keeps attach, holds and -// teardown. A send is the one mutation that may need those first: it makes sure the session has -// an owner as a step of its own serialized admission, see `structured-agent-session-send-preparation`. +// they share one path here rather than five copies in the host. The host keeps attach and teardown. +// Each opens the conversation first. A send, a Stop and an option pick are conversation writes, +// admitted without the writer lease; the delivery loop starts the provider child a send needs, and +// an operation only the provider can perform starts it before admission. import type { AgentJournalItemIdentity, @@ -15,21 +16,24 @@ import type { AgentSessionMutationEnvelope, AgentSessionMutationResult, AgentSessionOptionResult, - AgentSessionOptionsResult, AgentSessionPromptResult, AgentSessionSendResult, AgentSessionThreadGoalChange, AgentSessionThreadGoalResult } from '../../../shared/agent-session-wire' -import type { StructuredAgentSessionHolds } from './structured-agent-session-holds' +import { DISPATCH_REJECTED_CANCELLED } from '../../../shared/structured-agent-session-dispatch-rejection' import type { AgentSessionPromptRequest } from './structured-agent-session-turns-prompt' import { threadGoalPlan } from './structured-agent-session-thread-goal' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { admitAndRunAgentSessionMutation, - type AgentSessionMutationRequest + type AgentSessionMutationRequest, + type AgentSessionMutationSessionPreparation } from './structured-agent-session-mutation-admission' import { - prepareStructuredAgentSessionSend, + openForWrite, + openWithAgent, + sendPreparation, structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { @@ -44,19 +48,27 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + readStructuredAgentSessionOptions, + recordStructuredAgentSessionOptionIntent +} from './structured-agent-session-options-read' export type StructuredAgentSessionMutationContext = { deps: StructuredAgentSessionHostDeps sessions: Map publish: (sessionId: string, journal: StructuredAgentSessionHostSession['journal']) => void flushStreamedEvents: (sessionId: string) => Promise - requireSession: (sessionId: string) => StructuredAgentSessionHostSession + /** The host's accessor, for a caller outside the session's serialize. */ + conversation: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise - /** A send that finds the owner gone brings it back through here, inside its own serialize. */ - holds: Pick - /** Makes a closed session's journal readable again, inside the caller's serialize, for a send - * the ledger answers without an owner. */ - restoreReadable: (sessionId: string) => Promise + /** The session's conversation, opened when closed; inside the caller's serialize. */ + openConversation: (sessionId: string) => Promise + /** Gives the session a provider child; inside the caller's serialize. */ + ensureAgent: (sessionId: string) => Promise + /** A message was accepted: the session's delivery loop hands it over. */ + wakeDelivery: (sessionId: string) => void + /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. */ + stopAgent: (sessionId: string) => Promise now: () => number } @@ -78,7 +90,7 @@ function mutate( prepareSession, publish: (journal) => context.publish(envelope.sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, - providerChildPhase: () => context.sessions.get(envelope.sessionId)?.providerChildPhase, + providerChildPhase: () => context.sessions.get(envelope.sessionId)?.child?.phase, now: () => context.now() }) ) @@ -101,12 +113,19 @@ export function sendStructuredAgentSessionTurn( params.envelope, { ...plan, - run: (ctx) => { + run: async (ctx) => { const blocked = structuredAgentSessionSendBlock(context.deps.store.getRecord(ctx.sessionId)) - return blocked ? Promise.resolve(blocked) : plan.run(ctx) + if (blocked) { + return blocked + } + const accepted = await plan.run(ctx) + if (accepted.ok) { + context.wakeDelivery(ctx.sessionId) + } + return accepted } }, - (ledger, record) => prepareStructuredAgentSessionSend(context, params.envelope, ledger, record) + sendPreparation(context, params.envelope) ) } @@ -131,7 +150,41 @@ export function cancelStructuredAgentSessionTurn( context.serialize(`compact-cancel:${sessionId}`, task) } : context - return mutate(cancellationContext, caller, params.envelope, cancelPlan(params)) + const plan = cancelPlan(params) + if (params.scope || params.prompt) { + return mutate( + cancellationContext, + caller, + params.envelope, + plan, + openForWrite(context, params.envelope) + ) + } + return mutate( + cancellationContext, + caller, + params.envelope, + { + ...plan, + run: async (ctx) => { + // Stop withdraws every queued message first, whatever the start or the child is doing. + const withdrawn = await ctx.journal.rejectQueuedSubmissions( + ctx.fence, + DISPATCH_REJECTED_CANCELLED + ) + const child = context.sessions.get(ctx.sessionId)?.child + if (child?.phase === 'starting') { + // A start that may never land is the one thing here Stop has to end; the chat stays. + await context.stopAgent(ctx.sessionId) + return { ok: true, value: { turnId: params.turnId, cancelled: true } } + } + return child + ? plan.run(ctx) + : { ok: true, value: { turnId: params.turnId, cancelled: withdrawn.length > 0 } } + } + }, + openForWrite(context, params.envelope) + ) } export function respondToStructuredAgentSessionPrompt( @@ -139,7 +192,13 @@ export function respondToStructuredAgentSessionPrompt( caller: StructuredAgentSessionCaller, params: AgentSessionPromptRequest & { envelope: AgentSessionMutationEnvelope } ): Promise> { - return mutate(context, caller, params.envelope, promptPlan(params)) + return mutate( + context, + caller, + params.envelope, + promptPlan(params), + openForWrite(context, params.envelope) + ) } export async function setStructuredAgentSessionOption( @@ -149,7 +208,26 @@ export async function setStructuredAgentSessionOption( ): Promise> { // Outside the queue: a pick made while the provider starts then queues behind what its start persists. await context.deps.adapter.awaitOptionWritable?.(params.envelope.sessionId) - return mutate(context, caller, params.envelope, setOptionPlan(params)) + const plan = setOptionPlan(params) + const atRest = () => !context.sessions.get(params.envelope.sessionId)?.child + return mutate( + context, + caller, + params.envelope, + { + ...plan, + // Read as the call is admitted: with no child running, the pick is a conversation write — + // intent the next start replays. A running child's pick is still its owner's to make. + get conversationWrite() { + return atRest() ? (true as const) : undefined + }, + run: (ctx) => + atRest() + ? recordStructuredAgentSessionOptionIntent(context.deps.store, ctx, params) + : plan.run(ctx) + }, + openForWrite(context, params.envelope) + ) } export function changeStructuredAgentSessionThreadGoal( @@ -157,38 +235,13 @@ export function changeStructuredAgentSessionThreadGoal( caller: StructuredAgentSessionCaller, params: { envelope: AgentSessionMutationEnvelope; change: AgentSessionThreadGoalChange } ): Promise> { - return mutate(context, caller, params.envelope, threadGoalPlan(params)) -} - -export function readStructuredAgentSessionOptions( - context: StructuredAgentSessionMutationContext, - sessionId: string -): Promise { - return context.serialize(sessionId, async () => { - const session = context.requireSession(sessionId) - if (!context.deps.adapter.readOptions) { - throw new Error('structured_agent_session_options_unsupported') - } - const options = await context.deps.adapter.readOptions({ sessionId, fence: session.fence }) - return { - ...options, - rewind: - context.deps.store.getRecord(sessionId)?.rewind?.phase === 'prepared' || - context.deps.store.getRecord(sessionId)?.rewind?.phase === 'provider-succeeded' - ? { supported: false, reason: 'outcome-unknown' } - : (context.deps.adapter.rewindSupport?.(sessionId) ?? { - supported: false, - reason: 'unsupported' - }), - conversationCommands: context.deps.adapter.compact ? ['clear', 'compact'] : ['clear'], - ...(context.deps.adapter.supportsThreadGoal?.(sessionId) - ? { threadGoal: { current: session.journal.threadGoal() } } - : {}), - ...(context.deps.adapter.recordsContextUsage?.(sessionId) - ? { contextUsage: { current: session.journal.contextUsage() } } - : {}) - } - }) + return mutate( + context, + caller, + params.envelope, + threadGoalPlan(params), + openWithAgent(context, params.envelope) + ) } /** Settle provider-proven delivery independently of an in-flight client mutation. */ @@ -203,6 +256,7 @@ export async function settleStructuredAgentSessionLateDispatch( if (!session) { return } + const fence = structuredAgentSessionConversationFence(context.deps.store, input.sessionId) // The journal queue drains before close; the host queue would defer this past teardown. await session.journal.resolveDispatch( 'providerIdentity' in input @@ -210,13 +264,13 @@ export async function settleStructuredAgentSessionLateDispatch( clientMessageId: input.clientMessageId, state: 'accepted', providerIdentity: input.providerIdentity, - fence: session.fence + fence } : { clientMessageId: input.clientMessageId, state: 'rejected', reason: input.reason, - fence: session.fence + fence } ) } @@ -234,7 +288,9 @@ export async function settleStructuredAgentSessionLateDispatch( * it never makes a send re-deliverable, because the provider may well have run it. */ export async function releaseStructuredAgentSessionUnansweredDispatches( - context: Pick, + context: Pick & { + deps: { store: Pick } + }, input: { sessionId: string; reason: string } ): Promise { const session = context.sessions.get(input.sessionId) @@ -253,7 +309,7 @@ export async function releaseStructuredAgentSessionUnansweredDispatches( state: 'unknown', // The earlier reason names a sharper fact than this one does. reason: entry.reason ?? input.reason, - fence: session.fence, + fence: structuredAgentSessionConversationFence(context.deps.store, input.sessionId), recovered: true }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts index 8a18040daf35..faed6df6524b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts @@ -1,4 +1,5 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { adapterSupportsRecord } from './structured-agent-session-provider-support' /** * Chat-tab visibility is the deletion funnel: every path that removes a chat as a user-facing @@ -37,15 +38,26 @@ export type StructuredAgentSessionTab = { agent: AgentSessionRecord['provider'] } -export function listStructuredAgentSessionTabs( - sessions: ReadonlyMap< - string, - { params: { location: { workspaceId: string }; provider: AgentSessionRecord['provider'] } } - > +/** Tab existence from durable state: every requested id with a record this host serves, whether + * or not its journal opens. A chat that cannot be read keeps its tab and explains itself there. */ +export function listPersistedStructuredAgentSessionTabs( + deps: { + store: { getRecord: (sessionId: string) => AgentSessionRecord | null } + adapter: Parameters[0] + }, + sessionIds: readonly string[] ): StructuredAgentSessionTab[] { - return [...sessions.entries()].map(([sessionId, session]) => ({ - sessionId, - workspaceId: session.params.location.workspaceId, - agent: session.params.provider - })) + const tabs = new Map() + for (const sessionId of sessionIds) { + const record = deps.store.getRecord(sessionId) + if (!record || tabs.has(sessionId) || !adapterSupportsRecord(deps.adapter, record)) { + continue + } + tabs.set(sessionId, { + sessionId, + workspaceId: record.location.workspaceId, + agent: record.provider + }) + } + return [...tabs.values()] } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts index d04e17d2d26c..5fd0440a1fe1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.test.ts @@ -5,7 +5,7 @@ describe('structured agent-session host teardown', () => { it('names every phase, so the quit-path order is pinned rather than incidental', () => { const noop = async (): Promise => undefined const phases = structuredAgentSessionHostTeardownPhases({ - holds: { dispose: noop }, + idleSweep: { dispose: noop }, runtimeState: { stopLeaseRenewal: () => undefined, flushAllEventSinks: noop }, tasks: { drainAttaches: noop }, evictOwnedSessions: noop, @@ -14,7 +14,7 @@ describe('structured agent-session host teardown', () => { }) expect(phases.map((phase) => phase.name)).toEqual([ 'begin-resume-markers', - 'dispose-holds', + 'dispose-idle-sweep', 'stop-lease-renewal', 'drain-attaches', 'evict-owned-sessions', @@ -30,7 +30,7 @@ describe('structured agent-session host teardown', () => { const cleaned = vi.fn(async () => {}) const flush = vi.fn(async () => cleaned()) const phases = structuredAgentSessionHostTeardownPhases({ - holds: { dispose: cleaned }, + idleSweep: { dispose: cleaned }, runtimeState: { stopLeaseRenewal: () => {}, flushAllEventSinks: flush }, tasks: { drainAttaches: cleaned }, evictOwnedSessions: cleaned, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts index 6b1693c3dca2..33149381c81d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts @@ -9,6 +9,7 @@ import type { AgentSessionResumeTrigger } from '../../../shared/agent-session-resume-marker' import type { StructuredAgentSessionRestartResume } from './structured-agent-session-restart-resume-host' import { + abandonQueuedStructuredAgentSessionMessages, evictOwnedStructuredAgentSessions, type StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' @@ -46,7 +47,7 @@ async function withPhaseTimeout(run: () => Promise, timeoutMs: number): Pr /** The quit-path phase order, which is load-bearing rather than incidental. */ export function structuredAgentSessionHostTeardownPhases(collaborators: { - holds: { dispose: () => Promise | void } + idleSweep: { dispose: () => Promise | void } runtimeState: { stopLeaseRenewal: () => void flushAllEventSinks: () => Promise @@ -68,7 +69,7 @@ export function structuredAgentSessionHostTeardownPhases(collaborators: { } } }, - { name: 'dispose-holds', run: () => collaborators.holds.dispose() }, + { name: 'dispose-idle-sweep', run: () => collaborators.idleSweep.dispose() }, { name: 'stop-lease-renewal', run: () => collaborators.runtimeState.stopLeaseRenewal() }, { name: 'drain-attaches', run: () => collaborators.tasks.drainAttaches() }, { @@ -93,6 +94,8 @@ export async function tearDownStructuredAgentSessionHost(input: { sessions: Map retainSessionIds?: ReadonlySet acknowledgeSessionRelease?: (sessionId: string) => void + /** Quit closes every conversation, so it settles what they still queue as a close does. */ + abandonQueued?: (sessionId: string, session: StructuredAgentSessionHostSession) => Promise }): Promise { const failures: unknown[] = [] for (const phase of input.phases) { @@ -107,7 +110,12 @@ export async function tearDownStructuredAgentSessionHost(input: { ([sessionId]) => !input.retainSessionIds?.has(sessionId) ) // `allSettled`, so one rejected close cannot skip the others. - const closed = await Promise.allSettled(entries.map(([, session]) => session.journal.close())) + const closed = await Promise.allSettled( + entries.map(async ([sessionId, session]) => { + await input.abandonQueued?.(sessionId, session) + await session.journal.close() + }) + ) closed.forEach((result, index) => { const sessionId = entries[index]?.[0] if (result.status === 'fulfilled') { @@ -133,7 +141,7 @@ export async function tearDownStructuredAgentSessionHost(input: { export async function flushStructuredAgentSessionHost( context: StructuredAgentSessionLifetimeContext & - Pick[0], 'holds' | 'tasks'> & { + Pick[0], 'idleSweep' | 'tasks'> & { restartResume: StructuredAgentSessionRestartResume serialize: (sessionId: string, task: () => Promise) => Promise trigger: AgentSessionResumeTrigger @@ -160,6 +168,8 @@ export async function flushStructuredAgentSessionHost( sessions: context.sessions, retainSessionIds, acknowledgeSessionRelease: (sessionId) => - context.deps.adapter.acknowledgeSessionRelease?.(sessionId) + context.deps.adapter.acknowledgeSessionRelease?.(sessionId), + abandonQueued: (sessionId, session) => + abandonQueuedStructuredAgentSessionMessages(context.deps, sessionId, session.journal) }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-test-harness.ts index e0ac9c00d88a..1e0d77b0d524 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-test-harness.ts @@ -107,7 +107,7 @@ async function seedApproval(optionId = 'allow'): Promise<{ itemId: string; revis }) await host.flushStreamedEvents(SESSION) const itemId = agentJournalItemKey(identity) - const page = host.history({ sessionId: SESSION, direction: 'tail' }) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) const appended = page.ok ? page.page.items.find((item) => item.itemId === itemId) : null if (!appended) { throw new Error('provider approval was not written to the journal') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index fd2c911c74e5..f00673e52a55 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -1,8 +1,10 @@ import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' +import type { AgentSessionSavedStatusStore } from '../../runtime/agent-session-saved-status-store' import type { AgentSessionSpawnTokenScan } from '../../runtime/agent-session-spawn-token-process-scan' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { @@ -18,7 +20,7 @@ export type StructuredAgentSessionCaller = { callerKey: string } /** What the host believes about a session it just made addressable again. The workspace and agent * come from the record, so a caller publishes the host's view rather than a client's assertion. * `readable` is false when the journal could not be opened — the tab is still worth publishing, - * because attach recovers what read restore cannot. */ + * because the chat shows that failure and its Retry. */ export type StructuredAgentSessionReveal = { sessionId: string workspaceId: string @@ -26,25 +28,64 @@ export type StructuredAgentSessionReveal = { readable: boolean } +/** Which provider child: the adapter acquisition and the lease fence it writes at. */ +export type StructuredAgentSessionProviderChildIdentity = { + readonly generation: string | null + readonly fence: number +} + +/** The provider process behind a conversation. Written only in + * `structured-agent-session-provider-child`. */ +export type StructuredAgentSessionProviderChild = StructuredAgentSessionProviderChildIdentity & { + /** A publish-first acquire is `starting` until the adapter's `started` event; only then are its + * reported options fact. */ + phase: StructuredAgentSessionProviderChildPhase + /** The queued message whose delivery started this child, fixed when the start is made; absent + * for any other start. In memory only: it tells a restart offer its own start from another. */ + readonly startedFor?: string +} + +/** What ending a child established about its provider root. A stop's comes only from + * `stopAgentSessionProviderRoot`; an observed exit's root is gone by definition. */ +export type StructuredAgentSessionStopVerdict = { rootGone: boolean } + +export type StructuredAgentSessionChildEndCause = + | 'user-stop' + | 'host-stop' + | 'exit' + | 'attach-failed' + | 'evict' + +/** How the conversation's last child ended. In memory only: the delivery loop reads it to tell a + * Stop from a failure. */ +export type StructuredAgentSessionEndedChild = StructuredAgentSessionProviderChildIdentity & + StructuredAgentSessionStopVerdict & { + /** `user-stop` is a Stop the user asked for; `host-stop` is the host stopping the child for a + * cause of its own, which fails the start the delivery loop was waiting on. */ + cause: StructuredAgentSessionChildEndCause + /** Descriptive text only — the provider's diagnostic, or the host's cause. Decides nothing. */ + reason: string | null + duringStartup: boolean + startedFor?: string + /** Where the conversation's journal stood when the child ended, to order the end against a + * message's acceptance. */ + endedAt: AgentJournalCursor + } + +/** The conversation: its journal, params and readers outlive any child that serves it. */ export type StructuredAgentSessionHostSession = { /** Readonly: a new handle enters only through the session map's `set`, which binds its delivery. */ readonly journal: AgentSessionJournal params: AgentSessionAttachParams - fence: number - /** Whether THIS host generation is running the provider process behind the session. A journal - * restored for reading has none — so it may not be evicted to free a child, nor have its lease - * released as an observed exit. */ - hasProviderChild: boolean - /** Whether the child behind `hasProviderChild` has proven its start. A publish-first acquire - * is `starting` until the adapter's `started` event; only then are its reported options fact. */ - providerChildPhase: StructuredAgentSessionProviderChildPhase + /** The child THIS host generation runs for the conversation. A conversation opened for reading + * has none — so it may not be evicted to free a child, nor have its lease released as an + * observed exit. */ + child: StructuredAgentSessionProviderChild | null /** The wind-down this host still owes for a child it started: settling that generation's work - * and handing the lease back. A separate fact from `hasProviderChild`, which goes false the - * moment the adapter proves the exit — an eviction that aborts after that point must still be - * able to finish the wind-down on the next close. */ - owesProviderChildWindDown?: boolean - /** Exact adapter acquisition behind `hasProviderChild`; retained after exit to fence recovery. */ - acquisitionGeneration: string | null + * and handing the lease back. Outlives `child`, which ends the moment the adapter proves the + * exit — an eviction that aborts after that point must still finish it on the next close. */ + owesProviderChildWindDown?: StructuredAgentSessionProviderChildIdentity + lastEndedChild?: StructuredAgentSessionEndedChild } export type StructuredAgentSessionHostDeps = { @@ -70,8 +111,12 @@ export type StructuredAgentSessionHostDeps = { provider: AgentSessionRecord['provider'] ) => Promise | undefined> | Record | undefined now?: () => number - /** How long a session outlives its last surface. Tests drive this; production takes the default. */ - releaseGraceMs?: number + /** The idle sweep's period and window. Tests drive these; production takes the defaults. */ + idleSweep?: { intervalMs?: number; idleMs?: number } + /** Waits before each retry of a startup reconcile that did not complete; tests shorten them. */ + startupReconcileRetryDelaysMs?: readonly number[] + /** Whether an orchestration dispatch still owns this session's worker; absent answers no. */ + hasOpenDispatch?: (record: AgentSessionRecord) => boolean onEventSinkError?: (input: { sessionId: string; error: unknown }) => void /** Every status projection this host publishes. `replay` marks a re-projection of state the host * already knew (restore, an arriving subscriber) rather than a fresh journal edge. */ @@ -83,6 +128,8 @@ export type StructuredAgentSessionHostDeps = { * removed from. Both production hosts pass one — the desktop and headless `orcad`; absent, * every reader of that store simply lists no structured session. */ statusSink?: StructuredAgentSessionStatusSink + /** Each chat's last settled status, for the startup pass; absent, every listed chat is opened. */ + savedStatus?: Pick /** Host model catalog surface; absent means every catalog read answers `unknown`. */ modelCatalog?: AgentModelCatalogService } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts index 3c32989a4f9a..889fa6b65d4d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts @@ -208,7 +208,7 @@ describe('cancel', () => { turnId: 'turn-1' }) expect(result).toMatchObject({ ok: true, value: { cancelled: true } }) - const page = host.history({ sessionId: SESSION, direction: 'tail' }) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(page.ok && page.page.items[0]?.body).toMatchObject({ kind: 'status', text: 'Cancellation requested.' @@ -332,7 +332,7 @@ describe('cancel', () => { refusal: { code: 'agent_session_operation_unknown' } }) expect(cancelTurn).toHaveBeenCalledTimes(1) - expect(host.history({ sessionId: SESSION, direction: 'tail' })).toMatchObject({ + expect(await host.history({ sessionId: SESSION, direction: 'tail' })).toMatchObject({ ok: true, page: { items: [ @@ -395,7 +395,7 @@ describe('respondToPrompt', () => { ...fields }) - const page = host.history({ sessionId: SESSION, direction: 'tail' }) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) const answered = page.ok ? page.page.items.find((item) => item.itemId === itemId) : null expect(answered).toMatchObject({ revision: 2, @@ -490,7 +490,7 @@ describe('respondToPrompt', () => { ...fields }) expect(result.ok).toBe(true) - const page = host.history({ sessionId: SESSION, direction: 'tail' }) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) const statusId = agentJournalItemKey({ provider: 'orca', clientMessageId: `${prompt.itemId}#delivery` @@ -520,7 +520,7 @@ describe('setOption', () => { }) expect(setOption).toHaveBeenCalledTimes(1) expect(store.getRecord(SESSION)?.options).toEqual({ model: 'gpt-5', effort: 'high' }) - const page = host.history({ sessionId: SESSION, direction: 'tail' }) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(page.ok && page.page.items).toHaveLength(0) }) @@ -595,20 +595,20 @@ describe('restart', () => { acquire.mockClear() const listRecords = vi.spyOn(store, 'listRecords') - await host.restoreReadableSessions() + await host.restoreStartupSessions() const restoreReads = listRecords.mock.calls.length - await host.restoreReadableSessions() + await host.restoreStartupSessions() - expect(host.listSessionTabs()).toEqual([ + expect(host.listPersistedSessionTabs([SESSION])).toEqual([ { sessionId: SESSION, workspaceId: 'workspace-1', agent: 'codex' } ]) - const history = host.history({ sessionId: SESSION, direction: 'tail' }) + const history = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(history.ok && history.page.items).not.toHaveLength(0) expect(acquire).not.toHaveBeenCalled() expect(listRecords).toHaveBeenCalledTimes(restoreReads) }) - it('clears a stale conflicted recovery at restart, and reacquires the native owner when a surface holds it', async () => { + it('clears a stale conflicted recovery at restart, and reacquires the native owner on the next start', async () => { await attach() await store.transitionHandoff(SESSION, (record) => ({ ...record, @@ -622,10 +622,11 @@ describe('restart', () => { await reboot(async () => ({ outcome: 'pid-absent' })) acquire.mockClear() - await host.restoreReadableSessions() - // The recovery stage clears on evidence at startup; the child comes back only once a surface - // holds the session (see structured-agent-session-surface-lifetime.test.ts). - await host.hold(SESSION, 'surface-1') + await host.restoreStartupSessions() + // The recovery stage clears on evidence at startup; the child comes back only once work + // starts it — here the explicit attach a send's delivery would make. + const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + expect(await host.attach(CALLER, ensureParams(fence))).toMatchObject({ ok: true }) expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -644,7 +645,7 @@ describe('restart', () => { it('answers native for a chat whose start is still in flight', async () => { await attach() await reboot(async () => ({ outcome: 'pid-absent' })) - await host.restoreReadableSessions() + await host.restoreStartupSessions() const started = Promise.withResolvers() const release = Promise.withResolvers() const settled = acquire.getMockImplementation() @@ -657,12 +658,15 @@ describe('restart', () => { return settled(input) }) - const hold = host.hold(SESSION, 'surface-1') + const start = host.attach( + CALLER, + ensureParams(store.getRecord(SESSION)?.lease.runtimeFence ?? 0) + ) await started.promise const claimMidStart = store.getRecord(SESSION)?.lease.claimStatus const status = host.handoffStatus(SESSION) release.resolve() - await hold + await start // Mid-start the lease is only reserved; ownership does not wait for the agent. expect(claimMidStart).toBe('reserved') @@ -712,7 +716,7 @@ describe('subscribe', () => { it('opens with a snapshot and then streams cursor-qualified batches', async () => { await attach() const events: AgentSessionSubscribeEvent[] = [] - const dispose = host.subscribe({ + const dispose = await host.subscribe({ id: 'sub-1', sessionId: SESSION, emit: (event) => events.push(event) @@ -742,7 +746,7 @@ describe('subscribe', () => { } const events: AgentSessionSubscribeEvent[] = [] - host.subscribe({ + await host.subscribe({ id: 'sub-2', sessionId: SESSION, emit: (event) => events.push(event), @@ -763,14 +767,14 @@ describe('subscribe', () => { it('drops a failed transport without aborting the mutation or other subscribers', async () => { await attach() const events: AgentSessionSubscribeEvent[] = [] - host.subscribe({ + await host.subscribe({ id: 'dead-sub', sessionId: SESSION, emit: () => { throw new Error('socket closed') } }) - host.subscribe({ + await host.subscribe({ id: 'live-sub', sessionId: SESSION, emit: (event) => events.push(event) @@ -782,15 +786,25 @@ describe('subscribe', () => { body }) - expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) - expect(dispatch).toHaveBeenCalledTimes(1) - expect(events.some((event) => event.type === 'batch')).toBe(true) + expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // The failed transport does not stop the delivery loop either: the handover still lands and + // reaches the live subscriber. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => + expect( + events.some( + (event) => + event.type === 'batch' && + event.batch.submissions?.some((entry) => entry.dispatchState === 'accepted') + ) + ).toBe(true) + ) }) it('resets a subscriber whose epoch is gone', async () => { await attach() const events: AgentSessionSubscribeEvent[] = [] - host.subscribe({ + await host.subscribe({ id: 'sub-3', sessionId: SESSION, emit: (event) => events.push(event), @@ -802,7 +816,7 @@ describe('subscribe', () => { it('publishes the replacement fence when the owner generation changes', async () => { const record = await attach() const events: AgentSessionSubscribeEvent[] = [] - host.subscribe({ + await host.subscribe({ id: 'sub-4', sessionId: SESSION, emit: (event) => events.push(event) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 47a6d509ea8e..8ceb82a28e4d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -2,30 +2,36 @@ import type { AgentSessionRewindParams } from '../../../shared/agent-session-rew import { rewindStructuredAgentSession } from './structured-agent-session-rewind' import { StructuredConversationCommandController } from './structured-conversation-command-controller' // Structured agent-session host: where the lease, journal, and provider adapter meet. -// Mutations share one durable admission path and serialize per session. +// Mutations share one durable admission path and serialize per session. A conversation is reached +// only through `conversation`, which opens it at rest; an agent is started only by work that needs +// it, and the idle sweep is the one thing that puts it to rest. import type { AgentJournalSnapshot } from '../../../shared/agent-session-journal-types' import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' import type * as SessionWire from '../../../shared/agent-session-wire' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' import { createRestartReconciler } from './structured-agent-session-restart-reconcile' import type { AgentSessionSubscribeInput } from './structured-agent-session-subscribers' import { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' import * as providerSupport from './structured-agent-session-provider-support' -import { createStructuredAgentSessionHostRestore } from './structured-agent-session-reveal' +import { revealStructuredAgentSession } from './structured-agent-session-reveal' +import { + createStructuredAgentSessionHostStartupPass, + type StructuredAgentSessionStartupPass, + type StructuredAgentSessionStartupPriority +} from './structured-agent-session-restart-restore' import { structuredAgentSessionOwnerStatus } from './structured-agent-session-owner-status' import { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import { attachStructuredAgentSession } from './structured-agent-session-attach-orchestration' +import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import { - createStructuredAgentSessionHolds, - evictHeldStructuredAgentSession, - type StructuredAgentSessionLifetimeContext -} from './structured-agent-session-host-lifetime' -import type { - StructuredAgentSessionHolds, - StructuredAgentSessionHoldOptions -} from './structured-agent-session-holds' + ensureStructuredAgentSessionAgent, + ensureStructuredAgentSessionAgentForOperation +} from './structured-agent-session-agent-start' +import { + createStructuredAgentSessionConversationLifetime, + type StructuredAgentSessionConversationLifetime +} from './structured-agent-session-conversation-lifetime' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import * as sessionTabs from './structured-agent-session-host-tabs' import { @@ -38,7 +44,6 @@ import { flushStructuredAgentSessionHost } from './structured-agent-session-host import type { StructuredAgentSessionCaller, StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession, StructuredAgentSessionReveal } from './structured-agent-session-host-types' import { StructuredAgentSessionEventRecovery } from './structured-agent-session-event-recovery' @@ -50,6 +55,8 @@ import { type StructuredAgentSessionRestartResume } from './structured-agent-session-restart-resume-host' import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' +import { createStructuredAgentSessionConversationDelivery } from './structured-agent-session-host-delivery' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' export type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' export class StructuredAgentSessionHost { @@ -59,13 +66,16 @@ export class StructuredAgentSessionHost { ) private readonly sessions = new StructuredAgentSessionConversations({ deliver: (sessionId, journal) => this.subscribers.publish(sessionId, journal), - onDeliveryError: (sessionId, error) => this.deps.onEventSinkError?.({ sessionId, error }) + onDeliveryError: (sessionId, error) => this.deps.onEventSinkError?.({ sessionId, error }), + now: () => this.now() }) + // Every journal publish is activity: the one renewal the idle sweep reads. private readonly clientDelivery = new StructuredAgentSessionClientDelivery( this.sessions, () => this.now(), () => this.deps, - (sessionId) => this.holds.renew(sessionId) + (sessionId) => this.sessions.touch(sessionId), + (sessionId) => this.restartResume.onAgentStarted(sessionId) ) private readonly subscribers = this.clientDelivery.subscribers private readonly tasks = new StructuredAgentSessionTaskQueue() @@ -73,8 +83,11 @@ export class StructuredAgentSessionHost { private readonly reconcileLeases: ( sessionId: string ) => Promise - private readonly restore: ReturnType - private readonly holds: StructuredAgentSessionHolds + private readonly startupPass: StructuredAgentSessionStartupPass + private readonly lifetime: StructuredAgentSessionConversationLifetime + private readonly conversationDelivery: ReturnType< + typeof createStructuredAgentSessionConversationDelivery + > private readonly eventRecovery: StructuredAgentSessionEventRecovery private readonly backgroundTasks: StructuredAgentSessionBackgroundTaskChannel /** Public because the RPC surface addresses it directly; see the restart-resume collaborator. */ @@ -85,7 +98,7 @@ export class StructuredAgentSessionHost { deps, this.sessions, this.subscribers, - (sessionId) => this.requireSession(sessionId), + (sessionId) => this.lifetime.conversation(sessionId), this.clientDelivery.publishStatus ) this.runtimeState = new StructuredAgentSessionHostRuntimeState(deps, (sessionId, error) => @@ -95,23 +108,25 @@ export class StructuredAgentSessionHost { store: deps.store, probe: (record) => this.runtimeState.probeRecord(record), ...(deps.probeOwners ? { probeMany: deps.probeOwners } : {}), - now: () => this.now() + now: () => this.now(), + onReconciled: () => this.startupPass.onReconciled() }) - this.holds = createStructuredAgentSessionHolds( - () => this.attachContext(), - (sessionId) => this.close(sessionId) - ) - this.restore = createStructuredAgentSessionHostRestore(deps, this.sessions, () => this.now(), { - reconcile: this.reconcileLeases, - resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), + this.conversationDelivery = createStructuredAgentSessionConversationDelivery({ + deps, + sessions: this.sessions, serialize: (sessionId, task) => this.serialize(sessionId, task), - hasSession: this.hasSession, - // Site 10: cannot overwrite a live entry — the restorer returns early on - // `hasSession` inside the same serialized step as this `set`. - onReadable: (sessionId, restored) => { - this.sessions.set(sessionId, restored) - this.clientDelivery.publishRestored(sessionId) - } + // Quit drains a delivery start before it evicts, so the child it produces is stopped. + trackStart: (start) => this.tasks.trackAttach(start), + ensureProviderChild: (sessionId, startedFor) => + ensureStructuredAgentSessionAgent(this.attachContext(), sessionId, startedFor), + reset: (sessionId, journal, reset) => + this.subscribers.reset( + sessionId, + journal, + reset, + structuredAgentSessionConversationFence(deps.store, sessionId) + ), + publishRestored: this.clientDelivery.publishRestored }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -119,14 +134,14 @@ export class StructuredAgentSessionHost { sessions: this.sessions, flushLifecycle: (sessionId) => this.runtimeState.lifecycleBarrier(sessionId), publishFence: (sessionId, session) => - this.subscribers.snapshot(sessionId, session.journal, session.fence), + this.subscribers.snapshot( + sessionId, + session.journal, + structuredAgentSessionConversationFence(deps.store, sessionId) + ), publishStatus: this.clientDelivery.publishStatusAndSettlement, - hasResumeCapableHolder: (sessionId) => this.holds.hasResumeCapableHolder(sessionId), - restartReleaseGrace: (sessionId) => this.holds.renew(sessionId), - // Tracked: a quit drains a queued restart before it evicts, so no child outlives it. serialize: (sessionId, task) => this.tasks.trackAttach(this.serialize(sessionId, task)), now: () => this.now(), - ensureProviderChild: (id, options) => this.holds.ensureProviderChild(id, options), onBarrierError: (sessionId, error) => deps.onEventSinkError?.({ sessionId, error }) }) this.restartResume = createStructuredAgentSessionRestartResume( @@ -134,24 +149,31 @@ export class StructuredAgentSessionHost { this.sessions, structuredAgentSessionRestartResumeSurfaces(this, this.now) ) + this.lifetime = createStructuredAgentSessionConversationLifetime({ + context: () => this.lifetimeContext(), + sessions: this.sessions, + serialize: (sessionId, task) => this.serialize(sessionId, task), + open: (sessionId) => this.conversationDelivery.open(sessionId), + deliveryActive: (sessionId) => this.conversationDelivery.loop.isRunning(sessionId), + closeStatus: (sessionId, options) => this.clientDelivery.closeSession(sessionId, options) + }) + this.startupPass = createStructuredAgentSessionHostStartupPass(deps, { + sessions: this.sessions, + step: this.lifetime.startupStep, + seed: (sessionId, saved) => this.clientDelivery.seedStatus(sessionId, saved), + reconcile: () => this.reconcileLeases('startup'), + resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), + open: (sessionId) => this.conversationDelivery.open(sessionId), + now: () => this.now(), + disposed: this.lifetime.isDisposed + }) this.runtimeState.startLeaseRenewal() + this.lifetime.idleSweep.start() } private now = (): number => this.deps.now?.() ?? Date.now() hasSession = (sessionId: string): boolean => this.sessions.has(sessionId) - isHeld = (sessionId: string): boolean => this.holds.isHeld(sessionId) - - /** A surface bound to this session and wants it live. The FIRST hold on a session with no - * provider child is what resumes one; a retained hold (a subscription) only keeps it. */ - hold = ( - sessionId: string, - holderId: string, - options?: StructuredAgentSessionHoldOptions - ): Promise => this.holds.hold(sessionId, holderId, options) - - /** That surface is gone. The child outlives it by the idle window, and by any running turn. */ - release = (sessionId: string, holderId: string): void => this.holds.release(sessionId, holderId) handleAdapterEvent = (event: Parameters[0]) => this.eventRecovery.handle(event) @@ -162,7 +184,7 @@ export class StructuredAgentSessionHost { runtimeState: this.runtimeState, sessions: this.sessions, now: () => this.now(), - forgetStatus: this.clientDelivery.forgetStatus + publishStatus: this.clientDelivery.publishStatus } } @@ -174,29 +196,33 @@ export class StructuredAgentSessionHost { tasks: this.tasks, reconcileLeases: (sessionId) => this.reconcileLeases(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), - publishStatus: this.clientDelivery.publishStatus + publishStatus: this.clientDelivery.publishStatus, + openConversation: this.conversationDelivery.open } } - /** Releases a session's resources without ending the conversation: the record and journal stay - * on disk, so the same session can be attached again. */ - close(sessionId: string): Promise { - return this.serialize(sessionId, async () => { - await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId) - this.clientDelivery.closeSession(sessionId) - // The holders now look at a session that is gone; a failed eviction throws above, keeping them. - this.holds.forget(sessionId) - }) - } + /** Releases a session's resources without ending the conversation; see the lifetime's close. */ + close = (sessionId: string): Promise => this.lifetime.close(sessionId) supportsCreate = (location: AgentSessionExecutionLocation, agent: string): boolean => providerSupport.adapterSupportsCreate(this.deps.adapter, location, agent) - listSessionTabs = () => sessionTabs.listStructuredAgentSessionTabs(this.sessions) - getPersistedVisibleSessionTabIndex = () => this.deps.store.getVisibleSessionTabIndex() + /** The tabs the durable index lists, from records alone: no journal is read. */ + listPersistedSessionTabs = (sessionIds: readonly string[]) => + sessionTabs.listPersistedStructuredAgentSessionTabs(this.deps, sessionIds) + listVisibleSessionIds = (): string[] => this.deps.store.listVisibleSessionIds() getSessionTabId = (sessionId: string): string | null => this.deps.store.getSessionTabId(sessionId) - setSessionTabVisibility = (sessionId: string, visible: boolean, tabId?: string): Promise => - sessionTabs.setStructuredAgentSessionTabVisibility(this, sessionId, visible, tabId) + setSessionTabVisibility = async ( + sessionId: string, + visible: boolean, + tabId?: string + ): Promise => { + await sessionTabs.setStructuredAgentSessionTabVisibility(this, sessionId, visible, tabId) + // The tab edge of the row's lifetime; the handle close is the other. + if (!visible && !this.sessions.get(sessionId)?.child) { + this.clientDelivery.forgetStatus(sessionId) + } + } reconcileRestartLeases = async (): Promise => { const refusal = await this.reconcileLeases('startup') @@ -205,12 +231,13 @@ export class StructuredAgentSessionHost { } } - restoreReadableSessions = (sessionIds?: readonly string[]): Promise => - this.restore.restoreReadableSessions(sessionIds) + /** The one startup pass: list status from saved copies, then settle and open what that left. */ + restoreStartupSessions = (priority?: StructuredAgentSessionStartupPriority): Promise => + this.startupPass.run(priority) /** Make one persisted session addressable again; see `structured-agent-session-reveal`. */ revealSession = (sessionId: string): Promise => - this.restore.revealSession(sessionId) + revealStructuredAgentSession(this.deps, sessionId, (id) => this.lifetime.conversation(id)) private serialize = this.tasks.serialize.bind(this.tasks) @@ -227,9 +254,10 @@ export class StructuredAgentSessionHost { // Trigger inlined rather than imported: `AgentSessionResumeTrigger` in shared is the canonical // type, and this file has no line budget left for the import. async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise { + this.conversationDelivery.loop.dispose() await flushStructuredAgentSessionHost({ ...this.lifetimeContext(), - holds: this.holds, + idleSweep: this.lifetime, tasks: this.tasks, restartResume: this.restartResume, serialize: this.serialize, @@ -243,10 +271,13 @@ export class StructuredAgentSessionHost { sessions: this.sessions, publish: (sessionId, journal) => this.subscribers.publish(sessionId, journal), flushStreamedEvents: this.flushStreamedEvents, - requireSession: (sessionId) => this.requireSession(sessionId), + conversation: this.lifetime.conversation, serialize: (sessionId, task) => this.serialize(sessionId, task), - holds: this.holds, - restoreReadable: (sessionId) => this.restore.restoreReadableUnderSerialize(sessionId), + openConversation: this.conversationDelivery.open, + ensureAgent: (sessionId) => + ensureStructuredAgentSessionAgentForOperation(this.attachContext(), sessionId), + wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), + stopAgent: this.lifetime.stopAgent, now: () => this.now() } } @@ -282,10 +313,10 @@ export class StructuredAgentSessionHost { /** The fully reduced timeline, for readers that cannot tolerate a page's ambiguity — rows are * revised or tombstoned in place, so an item's ABSENCE from a bounded page proves nothing. */ - journalSnapshot = (sessionId: string): AgentJournalSnapshot => - this.requireSession(sessionId).journal.snapshot() + journalSnapshot = async (sessionId: string): Promise => + (await this.lifetime.conversation(sessionId)).journal.snapshot() - subscribe = (input: AgentSessionSubscribeInput): (() => void) => + subscribe = (input: AgentSessionSubscribeInput): Promise<() => void> => this.backgroundTasks.subscribe(input) settleLateDispatch = (input: Parameters[1]) => @@ -306,11 +337,13 @@ export class StructuredAgentSessionHost { /** Turns that settle from now on. Live-only: nothing missed is replayed. */ subscribeTurnCompletions = this.clientDelivery.subscribeTurnCompletions - private requireSession(sessionId: string): StructuredAgentSessionHostSession { - const session = this.sessions.get(sessionId) - if (!session) { - throw new Error(AGENT_SESSION_NOT_ATTACHED.code) - } - return session - } + /** Test rigs only: the collaborators the host builds itself, typed, for tests that drive them. */ + collaboratorsForTests = () => ({ + sessions: this.sessions, + subscribers: this.subscribers, + runtimeState: this.runtimeState, + conversationDelivery: this.conversationDelivery, + lifetime: this.lifetime, + serialize: this.serialize + }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts new file mode 100644 index 000000000000..5b9582da4bde --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts @@ -0,0 +1,292 @@ +// The idle sweep on a real host: what puts an agent to rest, what keeps it running, and what a +// reader and the session lists see when it does. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { + collectSubscriber, + createRestTestRig, + foundRestTestChat, + readerSaw, + IDLE_MS, + REST_TEST_CALLER as CALLER, + REST_TEST_SESSION as SESSION, + REST_TEST_THREAD as THREAD, + restTestSend, + sweepTicks, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { StructuredAgentSessionIdleSweep } from './structured-agent-session-idle-sweep' + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + await rig.dispose() +}) + +function providerEvents() { + const events = rig.adapter.acquire.mock.calls.at(-1)?.[0].events + if (!events) { + throw new Error('no provider child was started') + } + return events +} + +function lastStatus(): AgentSessionStatusSummary | undefined { + return rig.statusEvents + .flatMap((event) => (event.type === 'status' ? [event.session] : [])) + .findLast((summary) => summary.sessionId === SESSION) +} + +function fence(): number { + return rig.store.getRecord(SESSION)?.lease.runtimeFence ?? 1 +} + +describe('the idle sweep', () => { + it('stops an idle agent and keeps the conversation, its status row and its reader (P2-07)', async () => { + await foundRestTestChat(rig) + const reader = collectSubscriber() + await rig.host.subscribe({ id: 'reader', sessionId: SESSION, emit: reader.emit }) + rig.clock.now += IDLE_MS + 1 + + await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) + await vi.waitFor(() => expect(rig.store.getRecord(SESSION)?.lease.claimStatus).toBe('released')) + await vi.waitFor(() => expect(rig.adapter.acknowledgeSessionRelease).toHaveBeenCalledOnce()) + expect(rig.adapter.acknowledgeSessionRelease).toHaveBeenCalledWith(SESSION) + // Listed, idle, and no longer running on this host — never dropped from any list. + expect(lastStatus()).toMatchObject({ status: 'idle' }) + expect(lastStatus()?.hostExecutionOwned).toBeUndefined() + expect(rig.sink.forget).not.toHaveBeenCalled() + expect(reader.events.some((event) => event.type === 'end')).toBe(false) + + // Everything a chat at rest answers still answers, and the next send starts a new agent. + await expect(rig.host.readOptions(SESSION)).resolves.toMatchObject({ current: {} }) + const sent = await rig.host.send(CALLER, restTestSend('carry on', fence())) + expect(sent.ok).toBe(true) + await vi.waitFor(() => expect(rig.adapter.acquire).toHaveBeenCalledTimes(2)) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2)) + await vi.waitFor(() => expect(readerSaw(reader.events).texts).toContain('carry on')) + }) + + it('never stops an agent a message is queued for, and hands the message over (P2-08)', async () => { + await foundRestTestChat(rig) + rig.adapter.closeSession.mockClear() + // The loop takes the message and waits on the child's start, outside the lock. + const started = Promise.withResolvers() + const awaitStarted = vi.fn(() => started.promise) + Object.assign(rig.host.deps.adapter, { awaitStarted }) + const reader = collectSubscriber() + await rig.host.subscribe({ id: 'reader', sessionId: SESSION, emit: reader.emit }) + const sent = await rig.host.send(CALLER, restTestSend('queued one', fence())) + expect(sent.ok).toBe(true) + await vi.waitFor(() => expect(awaitStarted).toHaveBeenCalled()) + rig.clock.now += IDLE_MS + 1 + + await sweepTicks() + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + started.resolve() + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2)) + const settled = (await rig.host.journalSnapshot(SESSION)).submissions.at(-1) + expect(settled?.dispatchState).toBe('accepted') + const seen = readerSaw(reader.events).submissions + expect(seen.some((row) => row.dispatchState === 'accepted')).toBe(true) + expect( + seen.some((row) => row.dispatchState === 'rejected' || row.dispatchState === 'unknown') + ).toBe(false) + }) + + it('never stops an agent whose background work still runs (P2-09)', async () => { + await foundRestTestChat(rig) + // A Codex subagent thread is one of these tasks; the tracker projects both kinds alike. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a background roster fixture; the sweep reads only whether live tasks exist. + rig.adapter.backgroundTaskState.mockReturnValue({ + state: 'monitoring', + tasks: [ + { + taskId: 'subagent-1', + kind: 'subagent', + status: 'running', + title: 'reviewer', + startedAt: rig.clock.now + } + ] + } as never) + rig.clock.now += IDLE_MS + 1 + + await sweepTicks() + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + rig.adapter.backgroundTaskState.mockReturnValue(undefined) + rig.clock.now += IDLE_MS + 1 + await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) + }) + + // A finished child reads done before the lead's wake-up turn writes its first row; stopping the + // agent in that gap would lose the wake-up. Owed work is activity, as main's release clock had it. + it('gives an agent a full idle window after its background work ends', async () => { + await foundRestTestChat(rig) + rig.adapter.backgroundTaskState.mockReturnValue({ + state: 'monitoring', + tasks: [{ id: 'subagent-1', kind: 'agent', state: 'working' }] + }) + rig.clock.now += IDLE_MS + 1 + await sweepTicks() + rig.adapter.backgroundTaskState.mockReturnValue({ + state: 'monitoring', + tasks: [{ id: 'subagent-1', kind: 'agent', state: 'done' }] + }) + + await sweepTicks() + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + rig.clock.now += IDLE_MS + 1 + await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) + }) + + it('stops an agent whose roster holds only children that went idle or finished', async () => { + await foundRestTestChat(rig) + rig.adapter.backgroundTaskState.mockReturnValue({ + state: 'monitoring', + tasks: [ + { id: 'subagent-1', kind: 'agent', state: 'idle' }, + { id: 'command-1', kind: 'command', state: 'done' } + ] + }) + rig.clock.now += IDLE_MS + 1 + + await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) + }) + + it('never stops an agent while its lead turn runs, however quiet (P2-10)', async () => { + await foundRestTestChat(rig) + providerEvents().appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'working', ordinal: 50 }, + { kind: 'turn', turnId: 'working', state: 'running' } + ) + await rig.host.flushStreamedEvents(SESSION) + rig.clock.now += IDLE_MS + 1 + + await sweepTicks() + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + }) + + it('stops an agent whose chat is still open on screen (P2-12)', async () => { + await foundRestTestChat(rig) + const reader = collectSubscriber() + await rig.host.subscribe({ id: 'on-screen', sessionId: SESSION, emit: reader.emit }) + rig.clock.now += IDLE_MS + 1 + + await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) + expect(reader.events.some((event) => event.type === 'end')).toBe(false) + }) + + it('counts the idle window from the last activity (P2-13)', async () => { + await foundRestTestChat(rig) + rig.clock.now += IDLE_MS - 60_000 + // Activity at 29 minutes: a provider row reaching the journal. + providerEvents().appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 60 }, + { kind: 'status', text: 'still thinking' } + ) + await rig.host.flushStreamedEvents(SESSION) + rig.clock.now += 60_001 + + await sweepTicks() + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + rig.clock.now += IDLE_MS + await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) + }) + + it('never stops a worker whose orchestration dispatch is open, and stops it once it settles (P2-19 i)', async () => { + await rig.dispose() + let open = true + const hasOpenDispatch = vi.fn(() => open) + rig = await createRestTestRig({ hasOpenDispatch }) + await foundRestTestChat(rig) + rig.clock.now += 2 * IDLE_MS + + await sweepTicks(12) + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + expect(hasOpenDispatch).toHaveBeenCalledWith(expect.objectContaining({ sessionId: SESSION })) + open = false + rig.clock.now += IDLE_MS + 1 + await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) + }) + + it('keeps a child an unanswered prompt waits on (P2-22 i)', async () => { + await foundRestTestChat(rig) + providerEvents().appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'subagent-turn', ordinal: 70 }, + { + kind: 'approval', + title: 'Run the command?', + detail: null, + options: [{ id: 'allow', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } + ) + await rig.host.flushStreamedEvents(SESSION) + rig.clock.now += IDLE_MS + 1 + + await sweepTicks() + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + }) +}) + +describe('the idle sweep with no child running (P2-22 ii)', () => { + it('closes a handle whose only leftover is a prompt nobody can answer', async () => { + const journal = { + submissions: () => [], + pendingSubmissions: () => [], + snapshot: () => ({ + items: [ + { + itemId: 'prompt', + revision: 1, + sequence: 1, + observedAt: 0, + body: { + kind: 'approval', + title: 'Run?', + detail: null, + options: [], + resolution: { + state: 'pending', + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null + } + } + } + ] + }) + } + const sessions = Object.assign( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a session fixture carrying only the journal and child facts the sweep reads. + new Map([[SESSION, { journal, child: null } as never]]), + { lastActivityAt: () => 0, touch: () => undefined } + ) + const stopAgent = vi.fn(async () => undefined) + const closeConversation = vi.fn(async () => true) + const sweep = new StructuredAgentSessionIdleSweep({ + sessions, + serialize: (_id, task) => task(), + now: () => IDLE_MS + 1, + isDisposed: () => false, + deliveryActive: () => false, + backgroundTaskState: () => undefined, + hasOpenDispatch: () => false, + stopAgent, + stopStartingAgent: stopAgent, + closeConversation, + onError: (_id, error) => { + throw error + } + }) + await sweep.tick() + expect(stopAgent).not.toHaveBeenCalled() + expect(closeConversation).toHaveBeenCalledWith(SESSION) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts new file mode 100644 index 000000000000..3de0d66871e9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts @@ -0,0 +1,151 @@ +// The one thing that puts an idle agent to rest. +// +// Nothing a viewer does keeps an agent alive or starts one: a chat on screen and a chat in a +// background tab are the same to this sweep. Every few minutes it looks at each open conversation +// and stops the provider child of one that has been quiet for the idle window and owes no work, +// then drops the open journal handle of one that is only a cache. The conversation itself — its +// record, tab, status row and readers — is untouched, and the next send starts a new child. +// +// Owed work is derived on every tick, never stored, so there is nothing to disagree with it. + +import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' +import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-projection' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import type { AgentSessionBackgroundTaskState } from '../../../shared/agent-session-wire' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' + +export const STRUCTURED_AGENT_SESSION_IDLE_SWEEP_INTERVAL_MS = 5 * 60_000 +export const STRUCTURED_AGENT_SESSION_IDLE_MS = 30 * 60_000 + +export type StructuredAgentSessionIdleSweepDeps = { + sessions: ReadonlyMap & { + lastActivityAt: (sessionId: string) => number | undefined + touch: (sessionId: string) => void + } + serialize: (sessionId: string, task: () => Promise) => Promise + now: () => number + isDisposed: () => boolean + deliveryActive: (sessionId: string) => boolean + backgroundTaskState: (sessionId: string) => AgentSessionBackgroundTaskState | null | undefined + /** An orchestration dispatch that still owns this session's worker; derived from its database. */ + hasOpenDispatch: (sessionId: string) => boolean + /** Each of these runs inside the session's serialize and never takes it again. */ + stopAgent: (sessionId: string) => Promise + stopStartingAgent: (sessionId: string) => Promise + closeConversation: (sessionId: string) => Promise + onError: (sessionId: string, error: unknown) => void + intervalMs?: number + idleMs?: number +} + +/** A prompt the user has not answered. A subagent can raise one the lead turn cannot see. */ +export function hasPendingStructuredAgentSessionPrompt( + items: readonly AgentJournalRenderItem[] +): boolean { + return items.some( + (item) => + (item.body.kind === 'approval' || item.body.kind === 'question') && + item.body.resolution.state === 'pending' + ) +} + +export class StructuredAgentSessionIdleSweep { + private timer: ReturnType | null = null + private running = false + + constructor(private readonly deps: StructuredAgentSessionIdleSweepDeps) {} + + start(): void { + this.timer = setInterval( + () => void this.tick(), + this.deps.intervalMs ?? STRUCTURED_AGENT_SESSION_IDLE_SWEEP_INTERVAL_MS + ) + // An idle sweep must never be the reason a process stays alive at quit. + this.timer.unref?.() + } + + dispose(): void { + if (this.timer) { + clearInterval(this.timer) + this.timer = null + } + } + + /** One pass over every open conversation. Sessions run concurrently, so one waiting behind a + * locked start does not hold up the rest; a pass still running skips the next. */ + async tick(): Promise { + if (this.running || this.deps.isDisposed()) { + return + } + this.running = true + try { + await Promise.allSettled( + [...this.deps.sessions.keys()].map((sessionId) => + this.deps + .serialize(sessionId, () => this.tickUnderSerialize(sessionId)) + .catch((error: unknown) => this.deps.onError(sessionId, error)) + ) + ) + } finally { + this.running = false + } + } + + /** Re-derives everything inside the session's lock, so an accept already queued ahead is seen. */ + private async tickUnderSerialize(sessionId: string): Promise { + const session = this.deps.sessions.get(sessionId) + if (!session || this.deps.isDisposed()) { + return + } + // A stop that failed after the child was proven gone: finish it now, before the idle test, so + // the rows its settlement wrote cannot push the retry out. A message accepted since goes first. + if ( + session.owesProviderChildWindDown !== undefined && + !session.child && + !this.queuedOrDelivering(sessionId, session) + ) { + await this.deps.stopAgent(sessionId) + return + } + const lastActivityAt = this.deps.sessions.lastActivityAt(sessionId) ?? this.deps.now() + if (this.deps.now() - lastActivityAt < (this.deps.idleMs ?? STRUCTURED_AGENT_SESSION_IDLE_MS)) { + return + } + if (session.child) { + // A start that has been quiet this long is not coming: the host stops it, with its reason. + if (session.child.phase === 'starting') { + await this.deps.stopStartingAgent(sessionId) + return + } + if (this.owesWork(sessionId, session)) { + // Owed work is activity, so the agent gets a full window once it ends: a child can read + // done before the lead's wake-up turn writes anything. + this.deps.sessions.touch(sessionId) + return + } + await this.deps.stopAgent(sessionId) + } + await this.deps.closeConversation(sessionId) + } + + private queuedOrDelivering(sessionId: string, session: StructuredAgentSessionHostSession) { + return ( + this.deps.deliveryActive(sessionId) || + session.journal.submissions().some(isQueuedAgentJournalSubmission) + ) + } + + /** Work the running child still owes. Scoped to the child: with none, nothing here can pin the + * handle, and a leftover prompt or turn row is only history. */ + private owesWork(sessionId: string, session: StructuredAgentSessionHostSession): boolean { + const items = session.journal.snapshot().items + return ( + activeStructuredAgentSessionTurnId(items) !== null || + this.queuedOrDelivering(sessionId, session) || + agentChildWorkLiveness(this.deps.backgroundTaskState(sessionId)?.tasks) !== null || + this.deps.hasOpenDispatch(sessionId) || + hasPendingStructuredAgentSessionPrompt(items) + ) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts index ef08e1c7e93d..8dabd942705e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts @@ -15,11 +15,7 @@ import { journalDatabaseFile } from '../agent-session-journal/journal-paths' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' -import { - evictStructuredAgentSession, - STRUCTURED_AGENT_SESSION_EVICTION_STEPS, - type StructuredAgentSessionEvictionContext -} from './structured-agent-session-eviction' +import { closeStructuredAgentSessionConversationUnderSerialize } from './structured-agent-session-host-lifetime' import { tearDownStructuredAgentSessionHost } from './structured-agent-session-host-teardown' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' @@ -73,29 +69,7 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe return { journal, params: {} as StructuredAgentSessionHostSession['params'], - fence: 1, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null - } -} - -function evictionContext( - overrides: Partial -): StructuredAgentSessionEvictionContext { - return { - sessionId: SESSION, - hasProviderChild: false, - eventSink: { - drained: async () => ({ ok: true }) as const, - unbind: () => undefined, - close: () => undefined - } as unknown as StructuredAgentSessionEvictionContext['eventSink'], - adapter: {} as StructuredAgentSessionEvictionContext['adapter'], - forget: async () => undefined, - discardSink: () => undefined, - releaseLease: async () => undefined, - ...overrides + child: null } } @@ -140,46 +114,44 @@ describe('site 6: recovery rehydration', () => { }) }) -describe('sites 9 and 10: the delete and overwrite callbacks', () => { - it('awaits the journal close before dropping the map entry', async () => { +describe('sites 9 and 10: closing a conversation handle', () => { + it('drops the map entry before the close, and releases the handle', async () => { const journal = await journals.open({ identity: IDENTITY, journalDir }) const sessions = new Map([[SESSION, hostSession(journal)]]) const order: string[] = [] + const close = journal.close.bind(journal) + journal.close = async () => { + // A lock-free reader arriving now must find no entry, never a closing handle. + order.push(sessions.has(SESSION) ? 'close-while-indexed' : 'close-after-delete') + await close() + order.push('closed') + } - await evictStructuredAgentSession( - evictionContext({ - forget: async () => { - order.push('close-started') - await sessions.get(SESSION)?.journal.close() - order.push('closed') - sessions.delete(SESSION) - order.push('forgotten') - } - }), - STRUCTURED_AGENT_SESSION_EVICTION_STEPS - ) + await expect( + closeStructuredAgentSessionConversationUnderSerialize( + { sessions, closeStatus: () => order.push('status') }, + SESSION + ) + ).resolves.toBe(true) - expect(order).toEqual(['close-started', 'closed', 'forgotten']) + expect(order).toEqual(['status', 'close-after-delete', 'closed']) expect(sessions.size).toBe(0) await expectNothingHoldsTheDirectory(journalDir) }) - it('aborts the eviction with the session still indexed when the close rejects', async () => { + it('surfaces a rejected close to its caller', async () => { const journal = await journals.open({ identity: IDENTITY, journalDir }) const sessions = new Map([[SESSION, hostSession(journal)]]) + const close = journal.close.bind(journal) + journal.close = () => Promise.reject(new Error('close rejected')) await expect( - evictStructuredAgentSession( - evictionContext({ - forget: async () => { - await Promise.reject(new Error('close rejected')) - } - }), - STRUCTURED_AGENT_SESSION_EVICTION_STEPS + closeStructuredAgentSessionConversationUnderSerialize( + { sessions, closeStatus: () => undefined }, + SESSION ) - ).rejects.toMatchObject({ step: 'forget-session' }) - // Still indexed, so the next close is a real retry. - expect(sessions.has(SESSION)).toBe(true) + ).rejects.toThrow('close rejected') + journal.close = close }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts index 1874f7ee1c67..d2bfa1101482 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts @@ -60,8 +60,8 @@ function sendParams(text: string): { } } -function submissions(): unknown { - const state = host.history({ sessionId: SESSION, direction: 'tail' }) +async function submissions(): Promise { + const state = await host.history({ sessionId: SESSION, direction: 'tail' }) return state.ok ? state.page.submissions : null } @@ -71,6 +71,18 @@ function journal(): AgentSessionJournal { ).sessions.get(SESSION)!.journal } +/** A send is accepted first; this waits for the delivery loop to hand it to the provider. */ +async function handedOver(clientMessageId: string): Promise { + await vi.waitFor(async () => { + expect( + journal() + .submissions() + .find((entry) => entry.clientMessageId === clientMessageId)?.handedOverAt + ).toBeDefined() + expect(dispatch).toHaveBeenCalled() + }) +} + beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-wire-late-settle-')) resetHostTestOperationIds() @@ -126,14 +138,14 @@ describe('settling a send the provider proves it received after the ack window', }) ) const events: AgentSessionSubscribeEvent[] = [] - const unsubscribe = host.subscribe({ + const unsubscribe = await host.subscribe({ id: 'late-receipt', sessionId: SESSION, emit: (event) => events.push(event) }) const params = sendParams('echo before send completes') const pending = host.send(CALLER, params) - await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + await vi.waitFor(async () => expect(dispatch).toHaveBeenCalledTimes(1)) try { await host.settleLateDispatch({ sessionId: SESSION, @@ -152,10 +164,13 @@ describe('settling a send the provider proves it received after the ack window', finishDispatch({ state: 'unknown', reason: 'ack timeout' }) unsubscribe() } - await expect(pending).resolves.toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'accepted' } } - }) + await expect(pending).resolves.toMatchObject({ ok: true }) + // The late `unknown` from the handover does not reopen the proven acceptance. + await vi.waitFor(async () => + expect(await submissions()).toMatchObject([ + { clientMessageId: params.envelope.clientOperationId, dispatchState: 'accepted' } + ]) + ) await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } @@ -167,6 +182,10 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'unknown', reason: 'ack timeout' }) const params = sendParams('received just before shutdown') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) + await vi.waitFor(async () => + expect(await submissions()).toMatchObject([{ dispatchState: 'unknown' }]) + ) let settlement: Promise | undefined closeSession.mockImplementationOnce(async () => { settlement = host.settleLateDispatch({ @@ -181,15 +200,18 @@ describe('settling a send the provider proves it received after the ack window', await host.close(SESSION) await expect(settlement).resolves.toBeUndefined() await host.revealSession(SESSION) - expect(submissions()).toMatchObject([{ dispatchState: 'accepted' }]) + expect(await submissions()).toMatchObject([{ dispatchState: 'accepted' }]) expect(dispatch).toHaveBeenCalledTimes(1) }) it('moves a durable unknown to accepted so nothing offers to send it again', async () => { dispatch.mockRejectedValueOnce(new Error('socket closed')) const params = sendParams('sent while a turn was running') - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) + await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) + await vi.waitFor(async () => + expect(await submissions()).toMatchObject([{ dispatchState: 'unknown' }]) + ) await host.settleLateDispatch({ sessionId: SESSION, @@ -197,7 +219,7 @@ describe('settling a send the provider proves it received after the ack window', providerIdentity: { provider: 'claude', sessionId: THREAD, uuid: 'late-uuid' } }) - expect(submissions()).toMatchObject([ + expect(await submissions()).toMatchObject([ { clientMessageId: params.envelope.clientOperationId, dispatchState: 'accepted' } ]) // The point of the fix: the client stops rendering Retry, and Retry is what @@ -209,6 +231,7 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'admitted' }) const params = sendParams('queued behind the active turn') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) await host.settleLateDispatch({ sessionId: SESSION, @@ -217,7 +240,7 @@ describe('settling a send the provider proves it received after the ack window', reason: DISPATCH_REJECTED_CANCELLED }) - expect(submissions()).toMatchObject([ + expect(await submissions()).toMatchObject([ { clientMessageId: params.envelope.clientOperationId, dispatchState: 'rejected', @@ -230,6 +253,7 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'admitted' }) const params = sendParams('settle from provider echo') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) vi.spyOn(journal(), 'resolveDispatch').mockRejectedValueOnce( new Error('direct settlement write failed') ) @@ -247,7 +271,7 @@ describe('settling a send the provider proves it received after the ack window', { fence: store.getRecord(SESSION)?.lease.runtimeFence ?? 1 } ) - expect(submissions()).toMatchObject([ + expect(await submissions()).toMatchObject([ { clientMessageId: params.envelope.clientOperationId, dispatchState: 'accepted', @@ -259,6 +283,9 @@ describe('settling a send the provider proves it received after the ack window', it('leaves an already accepted send alone', async () => { const params = sendParams('ordinary send') await host.send(CALLER, params) + await vi.waitFor(async () => + expect(await submissions()).toMatchObject([{ dispatchState: 'accepted' }]) + ) await host.settleLateDispatch({ sessionId: SESSION, @@ -266,7 +293,7 @@ describe('settling a send the provider proves it received after the ack window', providerIdentity: { provider: 'claude', sessionId: THREAD, uuid: 'a-different-uuid' } }) - expect(submissions()).toMatchObject([ + expect(await submissions()).toMatchObject([ { clientMessageId: params.envelope.clientOperationId, dispatchState: 'accepted' } ]) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts index 300f1423cb3c..cd26b96e5cd1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts @@ -57,12 +57,19 @@ async function launchAndDeliver(): Promise<{ text: 'fix the failing test' }) const sent = await send.mock.results[0]!.value - return { - messageId, - dispatchState: sent.ok - ? sent.value.submission.dispatchState - : `refused:${sent.refusal.code}:${sent.refusal.message}` + if (!sent.ok) { + return { messageId, dispatchState: `refused:${sent.refusal.code}:${sent.refusal.message}` } } + // Accepted first; the delivery loop hands it over, and that outcome is what reached the agent. + let dispatchState = sent.value.submission.dispatchState + await vi.waitFor(async () => { + dispatchState = + (await host.journalSnapshot(created.value.sessionId)).submissions.find( + (entry) => entry.clientMessageId === sent.value.clientMessageId + )?.dispatchState ?? 'missing' + expect(dispatchState).not.toBe('pending') + }) + return { messageId, dispatchState } } beforeEach(() => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts index 42ffe172cd70..85ebc344610c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts @@ -25,12 +25,18 @@ import { const CALLER = { callerKey: 'client-1' } +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + let root: string let store: AgentSessionRecordStore let host: StructuredAgentSessionHost let acquire: Mock let probe: Mock<() => Promise> let stopOwnerProcess: Mock<(pid: number, signal: 'SIGTERM' | 'SIGKILL') => void> +let dispatch: Mock function openHost(): void { host = new StructuredAgentSessionHost({ @@ -39,7 +45,7 @@ function openHost(): void { acquire, closeSession: vi.fn(async () => true), releaseAcquisition: vi.fn(async () => true), - dispatch: vi.fn(async () => ({ state: 'admitted' as const })), + dispatch, cancelTurn: vi.fn(async () => ({ cancelled: false })), answerPrompt: vi.fn(async () => undefined), setOption: vi.fn(async () => undefined) @@ -56,6 +62,8 @@ function openHost(): void { /** Writes the lease an older build left behind, then starts a fresh app generation over it. */ async function persistFromOlderBuild(lease: Partial): Promise { expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + // Listed, as a chat with a tab is: the startup pass settles what a listed chat owes. + await store.setSessionTabVisibility(SESSION, true) const attached = store.getRecord(SESSION)?.lease await host.flushAllStreamedEvents() // Over the attached owner: the older build's stage or terminal owner kept it from releasing. @@ -65,6 +73,23 @@ async function persistFromOlderBuild(lease: Partial) openHost() } +/** A send is accepted at once; what became of it is the submission's state once the host's + * delivery settles it — handed over, or rejected with the reason the chat shows. */ +async function delivered(text: string) { + const sent = await send(text) + expect(sent).toMatchObject({ ok: true }) + const clientMessageId = sent.ok ? sent.value.clientMessageId : '' + const submission = async () => + (await host.journalSnapshot(SESSION)).submissions.find( + (candidate) => candidate.clientMessageId === clientMessageId + ) + await eventually(async () => { + const current = await submission() + expect(current?.dispatchState !== 'pending' || current?.handedOverAt !== undefined).toBe(true) + }) + return submission() +} + async function send(text: string) { const body = hostTestMessage(text) return host.send(CALLER, { @@ -87,6 +112,7 @@ beforeEach(async () => { resetHostTestOperationIds() probe = vi.fn(async () => ({ outcome: 'pid-absent' as const })) stopOwnerProcess = vi.fn() + dispatch = vi.fn(async () => ({ state: 'admitted' as const })) acquire = vi.fn(async ({ fence, spawnToken }) => ({ process: { hostId: 'local', pid: 4242, processStartTimeMs: NOW - 1_000, spawnToken }, link: { @@ -130,14 +156,15 @@ describe('a record an older build left mid terminal handoff', () => { claimStatus: 'released' }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() expect(store.getRecord(SESSION)?.lease).toMatchObject({ handoffStage: null, handoffOperationId: null, settlementRetryRequired: undefined }) - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', @@ -163,13 +190,14 @@ describe('a record an older build left mid terminal handoff', () => { }) expect(store.getRecord(SESSION)?.lease.handoffStage).toBe('recovering') - await host.restoreReadableSessions() + await host.restoreStartupSessions() expect(store.getRecord(SESSION)?.lease).toMatchObject({ handoffStage: null, handoffOperationId: null }) - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' }) }) @@ -182,13 +210,14 @@ describe('a record an older build left mid terminal handoff', () => { claimStatus: 'live' }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() expect(store.getRecord(SESSION)?.lease).toMatchObject({ handoffStage: null, claimStatus: 'released' }) - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', claimStatus: 'live' @@ -204,25 +233,33 @@ describe('a record an older build left mid terminal handoff', () => { }) probe.mockResolvedValue({ outcome: 'identity-matched', matchedOn: ['spawn-token'] }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'conflicted', handoffStage: 'manual-recovery' }) - expect(await send('while the terminal still runs')).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_conflict' } + // Accepted, then rejected by the start that cannot take the lease: the chat says why. + expect(await delivered('while the terminal still runs')).toMatchObject({ + dispatchState: 'rejected' }) + expect( + (await host.journalSnapshot(SESSION)).items.filter( + (item) => item.body.kind === 'status' && item.body.tone === 'error' + ) + ).toHaveLength(1) + expect(dispatch).not.toHaveBeenCalled() expect(stopOwnerProcess).not.toHaveBeenCalled() expect(acquire).not.toHaveBeenCalled() - // The user closes the terminal; the next open proves it gone and the chat takes over. + // The user closes the terminal; the next send's start proves it gone and the chat takes over. probe.mockResolvedValue({ outcome: 'pid-absent' }) - await host.hold(SESSION, 'surface-1') expect(stopOwnerProcess).not.toHaveBeenCalled() - expect(await send('after the terminal closed')).toMatchObject({ ok: true }) + expect(await delivered('after the terminal closed')).toMatchObject({ + dispatchState: 'pending' + }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', claimStatus: 'live', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index 62be89001596..24f341d28da2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -107,7 +107,8 @@ export async function admitAndRunAgentSessionMutation( envelope, hostFingerprint, now: request.now(), - ...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}) + ...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}), + ...(plan.conversationWrite ? { conversationWrite: true } : {}) }) if (!admitted) { return refuseAgentSessionMutation(AGENT_SESSION_NOT_ATTACHED) @@ -141,7 +142,8 @@ export async function admitAndRunAgentSessionMutation( envelope, hostFingerprint, ledger: { decision: 'admit', row: admission.row }, - lease: record.lease + lease: record.lease, + ...(plan.conversationWrite ? { conversationWrite: true } : {}) }) if (rerun.decision === 'refused') { return refuseAgentSessionMutation(rerun.refusal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts index 226881d38bfe..fd56944f3bd6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts @@ -29,8 +29,9 @@ export type MutationPlan = { method: string fields: Record operationIdScope?: 'global' + /** Admitted without the writer lease: see `admitAgentSessionMutation`. */ + conversationWrite?: true markUnknownBeforeRun?: boolean - beforeRun?: () => void run: (ctx: AgentSessionTurnContext) => Promise> replay: (ctx: AgentSessionTurnContext, outcome: AgentSessionOperationOutcome) => TValue | null rerunWhenReplayMissing?: (ctx: AgentSessionTurnContext) => boolean @@ -50,19 +51,22 @@ export function sendPlan(params: { return { method: 'agentSession.send', operationIdScope: 'global', + conversationWrite: true, markUnknownBeforeRun: true, // A control signal is not payload; it cannot alter durable replay. fields: { body: params.body }, - ...(params.beforeRun ? { beforeRun: params.beforeRun } : {}), recoverUnknownFromDurableState: true, // `retryUnknown` is a compatibility-only client signal. A recorded send // always replays and never reaches the provider twice. - run: (ctx) => - performSend(ctx, { + run: (ctx) => { + // Asked at acceptance: a send accepted after this one is queued behind it. + params.beforeRun?.() + return performSend(ctx, { clientMessageId, payloadFingerprint: params.envelope.payloadFingerprint, body: params.body - }), + }) + }, replay: (ctx, outcome) => { const submission = ctx.journal .submissions() @@ -101,6 +105,8 @@ export function cancelPlan(params: { }): MutationPlan { return { method: 'agentSession.cancel', + // Stop is a conversation write; a prompt or background-task cancel needs the live child. + ...(params.scope || params.prompt ? {} : { conversationWrite: true as const }), fields: { turnId: params.turnId, ...(params.scope ? { scope: params.scope } : {}), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts index b1eb5704f015..094ddffdcb27 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts @@ -2,7 +2,6 @@ import { join } from 'node:path' import { afterEach, expect, it, vi } from 'vitest' import { AgentSessionPreDispatchError, - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS, runSettledAgentSessionMutation } from './structured-agent-session-operation-settlement' import { @@ -42,6 +41,9 @@ async function context(): Promise { } } +/** Longer than any bookkeeping bound: a refusal must already be answered by then. */ +const SETTLED_WAIT_MS = 2_000 + afterEach(() => { vi.useRealTimers() vi.restoreAllMocks() @@ -80,7 +82,7 @@ it('returns a pre-dispatch refusal without waiting on redundant uncertainty pers }) try { await refusing.promise - await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS) + await vi.advanceTimersByTimeAsync(SETTLED_WAIT_MS) expect(returned).toBe(true) expect(writes).toHaveBeenCalledOnce() expect(hostTestState().dispatch).not.toHaveBeenCalled() @@ -129,9 +131,10 @@ it.each([1, 2])( } ) -// The pre-dispatch check judges only what the journal already holds; the send path never waits on -// the provider's stream barrier, so a sink that stalls or fails cannot delay or double a send. -it('dispatches without touching the event-stream barrier', async () => { +// A send's plan only accepts: it records the submission and never waits on the provider's stream +// barrier, so a sink that stalls or fails cannot delay or double a send. Handing it over is the +// delivery loop's. +it('accepts without touching the event-stream barrier or the provider', async () => { const ctx = await context() const { store } = hostTestState() vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue() @@ -149,42 +152,33 @@ it('dispatches without touching the event-stream barrier', async () => { }) expect(result).toMatchObject({ ok: true }) expect(beforeRun).toHaveBeenCalledOnce() - expect(hostTestState().dispatch).toHaveBeenCalledOnce() - expect(ctx.journal.submissions()[0]?.dispatchState).toBe('accepted') + expect(hostTestState().dispatch).not.toHaveBeenCalled() + expect(ctx.journal.submissions()[0]).toMatchObject({ + dispatchState: 'pending', + handoverRecorded: true + }) expect(barrier).not.toHaveBeenCalled() }) -it('refuses a superseded send without waiting on a stalled refusal write, and never dispatches late', async () => { +it('refuses a superseded send at acceptance, recording and dispatching nothing', async () => { const ctx = await context() const { store } = hostTestState() vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue() - const pending = Promise.withResolvers() - const refusing = Promise.withResolvers() - vi.spyOn(ctx.journal, 'resolveDispatch').mockImplementationOnce(() => { - refusing.resolve() - return pending.promise.then(() => ctx.journal.cursor()) - }) - vi.spyOn(console, 'warn').mockImplementation(() => {}) const beforeRun = vi.fn(() => { throw new AgentSessionPreDispatchError('agent_session_restart_work_superseded') }) const body = hostTestMessage('Continue the interrupted work') const operation = envelope('agentSession.send', { body }) vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - const result = runSettledAgentSessionMutation({ + const result = await runSettledAgentSessionMutation({ store, operationCallerKey: 'test', envelope: operation, context: ctx, plan: sendPlan({ envelope: operation, body, beforeRun }) }).catch((error: unknown) => error) - await refusing.promise - await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS) - expect(await result).toBeInstanceOf(AgentSessionPreDispatchError) - expect(hostTestState().dispatch).not.toHaveBeenCalled() - expect(ctx.journal.submissions()[0]?.dispatchState).toBe('pending') - pending.resolve() - await vi.advanceTimersByTimeAsync(0) + expect(result).toBeInstanceOf(AgentSessionPreDispatchError) + expect(ctx.journal.submissions()).toEqual([]) expect(hostTestState().dispatch).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts index baf871aef65e..5d5009aa918c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts @@ -11,8 +11,6 @@ export class AgentSessionPreDispatchError extends Error { } } -export const AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS = 2_000 - export async function runSettledAgentSessionMutation(input: { store: AgentSessionRecordStore operationCallerKey: string @@ -33,10 +31,7 @@ export async function runSettledAgentSessionMutation(input: { if (input.plan.markUnknownBeforeRun) { await settle({ status: 'unknown' }) } - outcome = await input.plan.run({ - ...input.context, - ...(input.plan.beforeRun ? { beforeDispatch: input.plan.beforeRun } : {}) - }) + outcome = await input.plan.run(input.context) await settle( outcome.ok ? (input.plan.settledOutcome?.(outcome.value) ?? { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts index 7957ed326a8e..3eae8a7ae08f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts @@ -152,6 +152,11 @@ describe('structured session options and close', () => { envelope: envelope('agentSession.send', { body }), body }) + // Accepted, then handed over by the delivery loop; the status is read once it answered. + await vi.waitFor(() => expect(dispatchedModels).toEqual([DEFAULT_MODEL])) + await vi.waitFor(async () => + expect((await host.journalSnapshot(SESSION)).submissions[0]?.dispatchState).toBe('accepted') + ) const events: AgentSessionStatusEvent[] = [] host.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) }) expect(events).toEqual([ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts new file mode 100644 index 000000000000..fd113dcdcfdb --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts @@ -0,0 +1,121 @@ +// A chat's options, live or at rest. +// +// At rest nothing here needs a child. The pick is the record's own `options` — what the next start replays — +// and the list is the host's model catalog, the same one the picker already reads before a chat +// exists. A pick made at rest is written to the record as intent, through the same transition a live +// pick takes, so the next start applies it. + +import type { + AgentSessionOptionResult, + AgentSessionOptionsResult +} from '../../../shared/agent-session-wire' +import { decodeStructuredAgentSessionOptionValue } from '../../../shared/structured-agent-session-option-codec' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { isClaudeStructuredOptionKey } from '../../claude/claude-structured-options' +import { isCodexTurnOptionKey } from '../../codex/codex-structured-turn-start' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' + +type RestingOptions = Pick + +async function readStructuredAgentSessionOptionsAtRest( + deps: Pick, + sessionId: string +): Promise { + const record = deps.store.getRecord(sessionId) + if (!record) { + throw new Error('agent_session_identity_required') + } + const catalog = (await deps.modelCatalog + ?.read({ agent: record.provider, sessionId }) + .catch(() => null)) ?? { origin: 'unknown' as const } + const models = catalog.origin === 'unknown' ? [] : catalog.models + const saved = record.options ?? {} + const fastMode = + saved.fastMode === undefined + ? null + : decodeStructuredAgentSessionOptionValue('fastMode', saved.fastMode) + return { + models, + ...(catalog.origin !== 'unknown' && catalog.fastModeSupport + ? { fastModeSupport: catalog.fastModeSupport } + : {}), + current: { + // An unknown model is one the client already treats as unconfirmed. + model: saved.model ?? models.find((model) => model.isDefault)?.id ?? '', + ...(saved.effort ? { effort: saved.effort } : {}), + ...(typeof fastMode === 'boolean' ? { fastMode } : {}) + } + } +} + +/** Records a pick for the next start. Only a key the provider would accept is kept. */ +export async function recordStructuredAgentSessionOptionIntent( + store: Pick, + ctx: Pick, + input: { key: string; value: string } +): Promise> { + const record = store.getRecord(ctx.sessionId) + const accepted = + record?.provider === 'codex' + ? isCodexTurnOptionKey(input.key) + : record?.provider === 'claude' && isClaudeStructuredOptionKey(input.key) + if (!record || !accepted) { + return { + ok: false, + refusal: { + code: 'agent_session_operation_invalid', + message: `${record?.provider ?? 'This session'} has no session option named ${input.key}` + } + } + } + const options = { ...record.options, [input.key]: input.value } + await ctx.persistOptions(options) + ctx.publish() + return { ok: true, value: { ...input, options } } +} + +/** A live child's own answer, or the answer at rest; the goal, usage and rewind either way. */ +export async function readStructuredAgentSessionOptions( + context: Pick< + StructuredAgentSessionMutationContext, + 'deps' | 'serialize' | 'openConversation' | 'conversation' + >, + sessionId: string +): Promise { + const { adapter, store } = context.deps + const live = await context.serialize(sessionId, async () => { + const session = await context.openConversation(sessionId) + const child = session?.child + if (!child) { + return null + } + if (!adapter.readOptions) { + throw new Error('structured_agent_session_options_unsupported') + } + return adapter.readOptions({ sessionId, fence: child.fence }) + }) + const options = live ?? (await readStructuredAgentSessionOptionsAtRest(context.deps, sessionId)) + // Re-acquired after the reads above: the handle they saw may have closed and reopened since. + const session = await context.conversation(sessionId) + const phase = store.getRecord(sessionId)?.rewind?.phase + const agent = session.params.provider + return { + ...options, + rewind: + phase === 'prepared' || phase === 'provider-succeeded' + ? { supported: false, reason: 'outcome-unknown' } + : (adapter.rewindSupport?.(sessionId, agent) ?? { + supported: false, + reason: 'unsupported' + }), + conversationCommands: adapter.compact ? ['clear', 'compact'] : ['clear'], + ...(adapter.supportsThreadGoal?.(sessionId, agent) + ? { threadGoal: { current: session.journal.threadGoal() } } + : {}), + ...(adapter.recordsContextUsage?.(sessionId, agent) + ? { contextUsage: { current: session.journal.contextUsage() } } + : {}) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts index d2d5d746469b..9da47ae1f22d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts @@ -1,10 +1,12 @@ -// Switching away from a chat starts the release clock, and the clock must never stop a provider -// child that still owes the user work. +// The idle sweep must never stop a provider child that still owes the user work. // // A Claude chat is published before its CLI answers initialize, and a message sent in that window -// is held until it does; evicting then refuses a message the user already sent. And a lead whose -// turn has settled can leave subagents, commands and monitors running inside the child; evicting -// then ends them silently. +// is accepted and stays queued until it does; the delivery loop hands it over once startup lands. +// The idle sweep ticks meanwhile. Inside the idle window it must leave that queued message and the +// starting child alone, and every journal publish — the handover included — starts the window +// again; only a chat quiet for the whole window loses its agent. And a lead whose turn has settled +// can leave subagents, commands and monitors running inside the child; stopping it then ends them +// silently. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -29,8 +31,8 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } -const SURFACE = 'desktop-chat:1' -const GRACE_MS = 5 +const SWEEP_MS = 5 +const IDLE_MS = 1_000 let root: string let store: AgentSessionRecordStore @@ -39,12 +41,14 @@ let adapter: ClaudeStructuredSessionAdapter let claude: ReturnType let landInit: () => void let lifecycle: Promise[] +let clock: number beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-owed-work-release-')) resetHostTestOperationIds() claude = fakeClaude() lifecycle = [] + clock = NOW const initLanded = new Promise((resolve) => { landInit = resolve }) @@ -65,7 +69,7 @@ beforeEach(async () => { lifecycle.push(host.handleAdapterEvent(mapped)) } }, - // As the runtime wires it: a held prompt's outcome reaches the journal out of band. + // As the runtime wires it: an admitted prompt's outcome reaches the journal out of band. onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), // Initialize answers only when the test says so. openConnection: async (launch, handlers) => { @@ -87,8 +91,8 @@ beforeEach(async () => { journalRoot: root, claimKeyId: 'key-1', mintSpawnToken: () => 'spawn-a', - releaseGraceMs: GRACE_MS, - now: () => NOW + idleSweep: { intervalMs: SWEEP_MS, idleMs: IDLE_MS }, + now: () => clock }) }) @@ -111,7 +115,6 @@ async function attachStarting(): Promise { }) ) expect(created).toMatchObject({ ok: true }) - await host.hold(SESSION, SURFACE) } async function send(text: string, dispatchState = 'pending'): Promise { @@ -133,38 +136,48 @@ async function send(text: string, dispatchState = 'pending'): Promise { return sent.ok ? sent.value.clientMessageId : '' } -function dispatchState(clientMessageId: string): string | undefined { - return host - .journalSnapshot(SESSION) - .submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState +async function dispatchState(clientMessageId: string): Promise { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + )?.dispatchState +} + +/** The delivery loop hands a message over on its own serialized steps after startup lands; this + * yields to them without moving the host clock. */ +async function untilSent(connection: { sent: unknown[] }): Promise { + for (let turn = 0; turn < 2000 && connection.sent.length === 0; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)) + } } -/** Long enough for several grace windows to elapse, so "not evicted" means the clock declined. */ -function waitOutSeveralGraceWindows(): Promise { - return new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) +/** Long enough for many sweep ticks, so "not stopped" means the sweep declined. */ +function waitOutSeveralSweeps(): Promise { + return new Promise((resolve) => setTimeout(resolve, SWEEP_MS * 20)) } -describe('a chat left while its Claude CLI is still starting', () => { - it('keeps the session for a message it is holding, and delivers it once startup lands', async () => { +describe('a Claude chat whose CLI is still starting', () => { + it('keeps a message queued inside the idle window, and delivers it once startup lands', async () => { await attachStarting() const held = await send('sent while starting') - host.release(SESSION, SURFACE) - await waitOutSeveralGraceWindows() + clock += IDLE_MS - 1 + await waitOutSeveralSweeps() expect(host.hasSession(SESSION)).toBe(true) expect(claude.connections[0].closeCount).toBe(0) - expect(dispatchState(held)).toBe('pending') + expect(await dispatchState(held)).toBe('pending') landInit() - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) - expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]) - await vi.waitFor(() => expect(dispatchState(held)).toBe('accepted')) + await vi.waitFor( + () => expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]), + { timeout: 3000 } + ) + await vi.waitFor(async () => expect(await dispatchState(held)).toBe('accepted')) }) - it('gives the message it wrote at startup a full grace to open its turn', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + it('gives the message it hands over at startup a full idle window to open its turn', async () => { await attachStarting() const held = await send('sent while starting') const connection = claude.connections[0] @@ -172,31 +185,34 @@ describe('a chat left while its Claude CLI is still starting', () => { connection.send = async (message) => { connection.sent.push(message) } - host.release(SESSION, SURFACE) - await vi.advanceTimersByTimeAsync(GRACE_MS * 3 - 1) + clock += IDLE_MS - 1 + await waitOutSeveralSweeps() expect(host.hasSession(SESSION)).toBe(true) - // Startup lands just before the clock's next tick. + // Startup lands just before the window closes; the handover's publish starts it again. landInit() - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) + await untilSent(connection) expect(connection.sent).toEqual([expect.objectContaining({ type: 'user' })]) - await vi.advanceTimersByTimeAsync(GRACE_MS - 1) + clock += IDLE_MS - 1 + await waitOutSeveralSweeps() expect(host.hasSession(SESSION)).toBe(true) expect(connection.closeCount).toBe(0) connection.handlers.onMessage?.(connection.sent[0]) await host.flushStreamedEvents(SESSION) - await vi.advanceTimersByTimeAsync(GRACE_MS * 3) + await waitOutSeveralSweeps() expect(host.hasSession(SESSION)).toBe(true) - expect(dispatchState(held)).toBe('accepted') + expect(await dispatchState(held)).toBe('accepted') }) - it('is released after the grace once its turn has finished', async () => { + it('stops the agent and closes the conversation once its turn has finished and it idled', async () => { await attachStarting() landInit() - await adapter.drainStartup(SESSION) - await send('answered', 'accepted') + await adapter.awaitStarted(SESSION) + const answered = await send('answered') + await vi.waitFor(async () => expect(await dispatchState(answered)).toBe('accepted')) claude.connections[0].handlers.onMessage?.({ type: 'result', subtype: 'success', @@ -206,25 +222,25 @@ describe('a chat left while its Claude CLI is still starting', () => { result: 'done' }) await host.flushStreamedEvents(SESSION) - expect(host.journalSnapshot(SESSION).submissions).toHaveLength(1) + expect((await host.journalSnapshot(SESSION)).submissions).toHaveLength(1) - host.release(SESSION, SURFACE) + clock += IDLE_MS await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) expect(claude.connections[0].closeCount).toBe(1) }) - it('is released after the grace when it owes nothing', async () => { + it('stops a start that stayed quiet for the whole window when it owes nothing', async () => { await attachStarting() - host.release(SESSION, SURFACE) + clock += IDLE_MS await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) expect(claude.connections[0].closeCount).toBe(1) }) }) -describe('a chat left while its settled lead still has background work running', () => { +describe('a chat whose settled lead still has background work running', () => { function frame(message: Record): void { claude.connections[0].handlers.onMessage?.({ session_id: PROVIDER_SESSION_ID, ...message }) } @@ -232,8 +248,9 @@ describe('a chat left while its settled lead still has background work running', async function settleTurnLeavingTask(taskType: string): Promise { await attachStarting() landInit() - await adapter.drainStartup(SESSION) - await send('fan out', 'accepted') + await adapter.awaitStarted(SESSION) + const fanOut = await send('fan out') + await vi.waitFor(async () => expect(await dispatchState(fanOut)).toBe('accepted')) frame({ type: 'system', subtype: 'task_started', @@ -253,11 +270,11 @@ describe('a chat left while its settled lead still has background work running', ['a subagent', 'local_agent'], ['a background command', 'local_bash'], ['a monitor', 'monitor'] - ])('keeps the session while %s runs, then releases it once that settles', async (_, type) => { + ])('keeps the agent while %s runs, then stops it once that settles', async (_, type) => { await settleTurnLeavingTask(type) - host.release(SESSION, SURFACE) - await waitOutSeveralGraceWindows() + clock += IDLE_MS + await waitOutSeveralSweeps() expect(host.hasSession(SESSION)).toBe(true) expect(claude.connections[0].closeCount).toBe(0) @@ -272,6 +289,7 @@ describe('a chat left while its settled lead still has background work running', // A finished background task can wake the lead; that turn is owed too until it settles. frame({ type: 'result', subtype: 'success', uuid: 'result-2', is_error: false, result: 'ok' }) await host.flushStreamedEvents(SESSION) + clock += IDLE_MS await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) expect(claude.connections[0].closeCount).toBe(1) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts index 0f115da5ebdc..83b1e138d09d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts @@ -12,6 +12,7 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach } from './structured-agent-session-attach-flow' const NOW = 1_800_000_000_000 @@ -86,6 +87,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -133,6 +135,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -172,6 +175,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-drift', claimKeyId: 'key-1', @@ -226,6 +230,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -297,6 +302,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts new file mode 100644 index 000000000000..805b1a473a87 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts @@ -0,0 +1,668 @@ +// The provider child is its own record on the conversation: stopping it, losing it or failing to +// start it ends the child, never the conversation. Against the real host, store and journal, with a +// live subscriber opened before each action. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import type { + AgentSessionStatusSummary, + AgentSessionSubscribeEvent +} from '../../../shared/agent-session-wire' +import { + DISPATCH_REJECTED_CANCELLED, + DISPATCH_REJECTED_PROVIDER_CLOSED +} from '../../../shared/structured-agent-session-dispatch-rejection' +import { providerStartupFailureOutcome } from './structured-agent-session-dead-generation-settlement' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { ensureStructuredAgentSessionAgent } from './structured-agent-session-agent-start' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { stopStructuredAgentSessionAgentUnderSerialize } from './structured-agent-session-host-lifetime' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let dispatch: Mock +let adapterExtras: Partial + +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +function generation(): string { + return `generation-${acquire.mock.calls.length}` +} + +const spawnChild: StructuredAgentSessionAdapter['acquire'] = async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + acquisitionGeneration: generation(), + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } +}) + +/** A Claude-shaped child: published at spawn, so it is `starting` until `started`. */ +const spawnStartingChild: StructuredAgentSessionAdapter['acquire'] = async (input) => ({ + ...(await spawnChild(input)), + providerChildPhase: 'starting' as const +}) + +function startHost(): void { + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: vi.fn(async () => ({ cancelled: false })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined), + ...adapterExtras + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + now: () => NOW + }) +} + +async function restartHost(): Promise { + await host.flushAllStreamedEvents() + startHost() +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-child-record-')) + resetHostTestOperationIds() + adapterExtras = {} + acquire = vi.fn(spawnChild) + dispatch = vi.fn(async (input) => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD, + turnId: `turn-${input.clientMessageId}`, + ordinal: dispatch.mock.calls.length + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + startHost() + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + await host.close(SESSION) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function sendParams(text: string) { + const body = hostTestMessage(text) + return { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } +} + +async function accept(text: string): Promise { + const params = sendParams(text) + const sent = await host.send(CALLER, params) + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + return params.envelope.clientOperationId +} + +function stop() { + const turnId = 'turn-none' + return host.cancel(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: { turnId } + }) + }, + turnId + }) +} + +async function submission(id: string): Promise { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === id + ) +} + +async function statusRows(): Promise<{ itemId: string; text: string; tone?: string }[]> { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' + ? [ + { + itemId: item.itemId, + text: item.body.text, + ...(item.body.tone ? { tone: item.body.tone } : {}) + } + ] + : [] + ) +} + +/** The child the conversation has now, as its lifecycle events name it. */ +function currentChild() { + const child = conversation()?.child + if (!child?.generation) { + throw new Error('no child indexed') + } + return { sessionId: SESSION, fence: child.fence, acquisitionGeneration: child.generation } +} + +function exit(child: ReturnType, reason: string, startupUnproven?: true) { + return host.handleAdapterEvent({ + type: 'ended', + ...child, + reason, + cause: 'unexpected-exit', + ...(startupUnproven ? { startupUnproven } : {}) + }) +} + +function rejectedIn(events: AgentSessionSubscribeEvent[], id: string): boolean { + return events.some( + (event) => + event.type === 'batch' && + event.batch.submissions.some( + (entry) => entry.clientMessageId === id && entry.dispatchState === 'rejected' + ) + ) +} + +function conversation() { + return host['sessions'].get(SESSION) +} + +async function subscribe(): Promise { + const events: AgentSessionSubscribeEvent[] = [] + await host.subscribe({ + id: 'sub-1', + sessionId: SESSION, + emit: (event) => events.push(structuredClone(event)) + }) + return events +} + +/** The chat's status row as a session list sees it, frame by frame. */ +function watchStatus(): AgentSessionStatusSummary[] { + const frames: AgentSessionStatusSummary[] = [] + host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status' && event.session.sessionId === SESSION) { + frames.push(event.session) + } + } + }) + return frames +} + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} + +describe('Stop on a child still proving its start', () => { + it('ends the child and keeps the conversation and its readers (R1)', async () => { + const ended = deferred() + adapterExtras = { + awaitStarted: vi.fn(() => ended.promise), + closeSession: vi.fn(async () => { + ended.resolve() + return true + }) + } + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + const first = await accept('hello') + const journal = conversation()?.journal + const events = await subscribe() + const frames = watchStatus() + await eventually(() => expect(conversation()?.child?.phase).toBe('starting')) + + expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + + // The same conversation: no reopen, and the chat told it is idle again. + expect(conversation()?.journal).toBe(journal) + expect(conversation()?.child).toBeNull() + expect(conversation()?.lastEndedChild).toMatchObject({ cause: 'user-stop', rootGone: true }) + expect(frames.at(-1)).not.toHaveProperty('hostExecutionPhase') + expect(frames.at(-1)).not.toHaveProperty('hostExecutionOwned') + expect(await submission(first)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + // A Stop is not a failure: no row, and the loop is gone. + expect(await statusRows()).toEqual([]) + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) + + const next = await accept('after stop') + await eventually(async () => expect((await submission(next))?.dispatchState).toBe('accepted')) + expect(conversation()?.journal).toBe(journal) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([next]) + // The reader opened before the Stop saw the next message delivered on the same stream. + expect( + events.some( + (event) => + event.type === 'batch' && + event.batch.submissions.some( + (entry) => entry.clientMessageId === next && entry.dispatchState === 'accepted' + ) + ) + ).toBe(true) + }) +}) + +describe('a settlement retry for an earlier child inside the attach for the next one', () => { + it('settles the earlier child and leaves the queued message to the new child (R1)', async () => { + // The earlier child's settlement is still owed; the lease is otherwise released. + await store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { + ...record.lease, + settlementRetryRequired: true, + settlementRetryId: `provider-exit:${SESSION}:1:generation-1` + } + })) + const retryFence = store.getRecord(SESSION)!.lease.runtimeFence + const id = await accept('for the next child') + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(store.getRecord(SESSION)?.lease.settlementRetryRequired).toBeUndefined() + // Handed over at the new child's fence, which the attach reserved after the retry. + const newFence = store.getRecord(SESSION)!.lease.runtimeFence + expect(newFence).toBeGreaterThan(retryFence) + expect((await submission(id))?.fence).toBe(newFence) + expect(conversation()?.child).toMatchObject({ generation: generation(), fence: newFence }) + }) +}) + +describe('a published child that dies while it proves its start', () => { + const EXIT = 'claude stream-json exited (code 1)' + const TEXT = providerStartupFailureOutcome(EXIT) + + it.each([['the loop sees the start fail first'], ['the exit is processed first']])( + 'leaves one error row keyed by the start, and every queued message rejected with it: %s (R2)', + async (order) => { + const settled = deferred() + adapterExtras = { awaitStarted: vi.fn(() => settled.promise) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + const first = await accept('first') + const events = await subscribe() + const second = await accept('second') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalled()) + const child = currentChild() + + if (order === 'the exit is processed first') { + await exit(child, EXIT, true) + settled.resolve(TEXT) + } else { + settled.resolve(TEXT) + await eventually(async () => + expect((await submission(second))?.dispatchState).toBe('rejected') + ) + await exit(child, EXIT, true) + } + + await eventually(async () => + expect((await submission(second))?.dispatchState).toBe('rejected') + ) + expect(await statusRows()).toEqual([ + { + itemId: `orca:${encodeURIComponent(`start-failure:${child.acquisitionGeneration}`)}`, + text: TEXT, + tone: 'error' + } + ]) + expect(await submission(first)).toMatchObject({ dispatchState: 'rejected', reason: TEXT }) + expect(await submission(second)).toMatchObject({ dispatchState: 'rejected', reason: TEXT }) + expect(rejectedIn(events, second)).toBe(true) + expect(dispatch).not.toHaveBeenCalled() + expect(acquire).toHaveBeenCalledTimes(2) + } + ) +}) + +/** A start by an operation that needs the agent, such as a goal change, outside the loop. */ +function startForOperation() { + return host['serialize'](SESSION, () => + ensureStructuredAgentSessionAgent(host['attachContext'](), SESSION) + ) +} + +describe('a start another operation made that dies while a sent message waits on it', () => { + const EXIT = 'claude stream-json exited (code 1)' + const TEXT = providerStartupFailureOutcome(EXIT) + + it("is the message's own failed start: one error row, the message rejected, no second start (R2)", async () => { + adapterExtras = { awaitStarted: vi.fn(async () => TEXT) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + // An operation that needs the agent starts a child that has not proven its start. + await startForOperation() + const operationChild = currentChild() + const events = await subscribe() + const params = sendParams('hello') + + // Accepted first; that child's exit is settled before the loop's first step. + const sent = host.send(CALLER, params) + const exited = exit(operationChild, EXIT, true) + expect(await sent).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await exited + const id = params.envelope.clientOperationId + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + await settleLoop() + expect((await submission(id))?.reason).toBe(TEXT) + expect(await statusRows()).toEqual([ + { + itemId: `orca:${encodeURIComponent(`start-failure:${operationChild.acquisitionGeneration}`)}`, + text: TEXT, + tone: 'error' + } + ]) + expect(rejectedIn(events, id)).toBe(true) + // The setup's child and the operation's: nothing started again into the same failure. + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('leaves a message sent after that start failed to a fresh start (R2)', async () => { + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await startForOperation() + await exit(currentChild(), EXIT, true) + expect(await statusRows()).toHaveLength(1) + + const id = await accept('after the failure') + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) + expect(acquire).toHaveBeenCalledTimes(3) + }) + + it('starts again for a message whose proven child crashed: only a failed start settles it (R2)', async () => { + await restartHost() + await startForOperation() + const params = sendParams('hello') + + const sent = host.send(CALLER, params) + const exited = exit(currentChild(), 'codex app-server crashed') + await sent + await exited + const id = params.envelope.clientOperationId + + await eventually(async () => expect((await submission(id))?.dispatchState).not.toBe('pending')) + expect((await submission(id))?.dispatchState).toBe('accepted') + expect(acquire).toHaveBeenCalledTimes(3) + }) + + it('waits on a child started since the failed one, not on the failure (R2)', async () => { + adapterExtras = { awaitStarted: vi.fn(async () => undefined) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + await startForOperation() + const params = sendParams('hello') + + // A second operation's start lands after the first child's exit, before the loop's first step. + const sent = host.send(CALLER, params) + const exited = exit(currentChild(), EXIT, true) + const held = startForOperation() + await sent + await exited + await held + const id = params.envelope.clientOperationId + + await eventually(async () => expect((await submission(id))?.dispatchState).not.toBe('pending')) + expect((await submission(id))?.dispatchState).toBe('accepted') + expect(acquire).toHaveBeenCalledTimes(3) + }) +}) + +describe('a child that ends before its message is handed over', () => { + it('starts one child for the message, then rejects it and stops (R2)', async () => { + // The child the loop starts exits between its start step and its handover step. + adapterExtras = { + awaitStarted: vi.fn(async () => { + await exit(currentChild(), 'codex app-server crashed') + }) + } + await restartHost() + const id = await accept('hello') + const events = await subscribe() + + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + expect((await submission(id))?.reason).toContain('codex app-server crashed') + expect(await statusRows()).toEqual([ + { itemId: expect.any(String), text: (await submission(id))?.reason, tone: 'error' } + ]) + expect(rejectedIn(events, id)).toBe(true) + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('another child indexed while the loop waits on the one it started', () => { + it('hands nothing over until the child now there has proven its start (R2)', async () => { + const starts = new Map>>() + const startOf = (generation: string) => { + const start = starts.get(generation) ?? deferred() + starts.set(generation, start) + return start + } + adapterExtras = { + awaitStarted: vi.fn(() => startOf(currentChild().acquisitionGeneration).promise), + // The stop ends the child without settling the start the loop is waiting on. + closeSession: vi.fn(async () => true) + } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + const first = await accept('first') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(1)) + const stopped = currentChild() + expect(await stop()).toMatchObject({ ok: true }) + const second = await accept('second') + // An operation that needs the agent starts its own child before the loop's handover step runs. + await host['serialize'](SESSION, () => + ensureStructuredAgentSessionAgent(host['attachContext'](), SESSION) + ) + const replacement = currentChild() + expect(replacement.acquisitionGeneration).not.toBe(stopped.acquisitionGeneration) + + startOf(stopped.acquisitionGeneration).resolve() + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(2)) + expect(dispatch).not.toHaveBeenCalled() + + await host.handleAdapterEvent({ + type: 'started', + ...replacement, + reportedOptions: { model: 'sonnet' }, + restoreSkippedOptions: [] + }) + startOf(replacement.acquisitionGeneration).resolve() + + await eventually(async () => expect((await submission(second))?.dispatchState).toBe('accepted')) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([second]) + expect(await submission(first)).toMatchObject({ reason: DISPATCH_REJECTED_CANCELLED }) + }) +}) + +describe('a quit with a message still queued', () => { + /** Read by the next launch, through the same open any reader takes. */ + async function afterRelaunch(id: string): Promise { + startHost() + await host.revealSession(SESSION) + return await submission(id) + } + + it('settles a message no child ever had the way a chat close does (R2)', async () => { + // Quit has begun — its first step stops the delivery loops — when this message is accepted. + host['conversationDelivery'].loop.dispose() + const id = await accept('hello') + expect(conversation()?.child).toBeNull() + await host.flushAllStreamedEvents() + + expect(await afterRelaunch(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + }) + + it('waits for the start already in flight and stops the child it produced (R2)', async () => { + const starting = deferred() + const closeSession = vi.fn(async () => true) + adapterExtras = { closeSession } + await restartHost() + // The loop's start step is under way, but has not reached its attach yet. + const resolveRecovery = host['runtimeState'].resolveRecovery.bind(host['runtimeState']) + const recovering = vi.spyOn(host['runtimeState'], 'resolveRecovery') + recovering.mockImplementationOnce(async (sessionId) => { + await starting.promise + return resolveRecovery(sessionId) + }) + const id = await accept('hello') + await eventually(() => expect(recovering).toHaveBeenCalled()) + + const quit = host.flushAllStreamedEvents() + starting.resolve() + await quit + + expect(closeSession).toHaveBeenCalledWith(SESSION) + expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released' }) + expect(dispatch).not.toHaveBeenCalled() + expect(await afterRelaunch(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + }) +}) + +describe('a send whose start failed, sent again with the same operation id', () => { + it('replays the recorded rejection and starts no second agent (R2)', async () => { + acquire.mockRejectedValueOnce(new Error('spawn claude ENOENT')) + const fenceBefore = store.getRecord(SESSION)!.lease.runtimeFence + const params = sendParams('hello') + // A failed start is a rejected message, never a refused send. + expect(await host.send(CALLER, params)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + const id = params.envelope.clientOperationId + await eventually(async () => expect((await submission(id))?.dispatchState).toBe('rejected')) + // The start moved the fence while the message was out. + expect(store.getRecord(SESSION)!.lease.runtimeFence).toBeGreaterThan(fenceBefore) + const starts = acquire.mock.calls.length + + const replay = await host.send(CALLER, params) + + expect(replay).toMatchObject({ + ok: true, + replayed: true, + value: { submission: { clientMessageId: id, dispatchState: 'rejected' } } + }) + await settleLoop() + expect(acquire).toHaveBeenCalledTimes(starts) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +async function settleLoop(): Promise { + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) +} + +describe('how a stopped child ends the start its loop was waiting on', () => { + /** A child the loop waits on, whose start the stop below does not settle, so a message sent + * after the stop is queued when the loop next looks. */ + async function stoppedWhileStarting(stop: () => Promise) { + const start = deferred() + adapterExtras = { + awaitStarted: vi.fn(() => start.promise), + closeSession: vi.fn(async () => true) + } + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await accept('first') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(1)) + await stop() + const second = await accept('second') + adapterExtras.awaitStarted = undefined + start.resolve() + return second + } + + it("goes on after a user's Stop and delivers what was sent since (R2)", async () => { + const second = await stoppedWhileStarting(async () => { + expect(await stop()).toMatchObject({ ok: true }) + }) + + await eventually(async () => expect((await submission(second))?.dispatchState).toBe('accepted')) + expect(conversation()?.lastEndedChild).toMatchObject({ cause: 'user-stop', reason: null }) + expect(await statusRows()).toEqual([]) + }) + + it('fails the start after a host stop, with the stop as the reason (R2)', async () => { + const reason = 'Claude never finished starting, so Orca stopped it.' + const second = await stoppedWhileStarting(() => + host['serialize'](SESSION, () => + stopStructuredAgentSessionAgentUnderSerialize(host['lifetimeContext'](), SESSION, { + cause: 'host-stop', + reason + }) + ) + ) + + await eventually(async () => expect((await submission(second))?.dispatchState).toBe('rejected')) + expect((await submission(second))?.reason).toBe(reason) + expect(await statusRows()).toEqual([ + { itemId: expect.any(String), text: reason, tone: 'error' } + ]) + expect(dispatch).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts new file mode 100644 index 000000000000..aeaae60c86bc --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -0,0 +1,74 @@ +// The provider child behind a conversation, kept as its own record on the conversation's entry. +// +// The entry is the conversation and outlives any number of children. A child enters only when an +// attach has fully succeeded, and leaves only through `endProviderChild`, which every ending shares: +// an exit, a failed re-attach, a Stop and an eviction. Each is matched on the child's generation +// and fence, so an ending that arrives late for an older child cannot end a newer one. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' + +type ChildBearer = Pick & { + journal: Pick +} + +/** The fence a conversation write carries: the record's, which is where the next child starts. A + * child's own writes carry `child.fence`, which equals it while that child holds the lease. */ +export function structuredAgentSessionConversationFence( + store: Pick, + sessionId: string +): number { + return store.getRecord(sessionId)?.lease.runtimeFence ?? 0 +} + +/** For the end of a successful attach only: a failed one never wrote a child to take back. */ +export function indexProviderChild( + session: ChildBearer, + child: StructuredAgentSessionProviderChild +): void { + session.child = child +} + +export function markProviderChildStarted( + session: ChildBearer, + identity: StructuredAgentSessionProviderChildIdentity +): boolean { + const child = matchingChild(session, identity) + if (child) { + child.phase = 'ready' + } + return child !== null +} + +export function endProviderChild( + session: ChildBearer, + ended: Omit +): boolean { + const child = matchingChild(session, ended) + if (!child) { + return false + } + session.child = null + session.lastEndedChild = { + ...ended, + ...(child.startedFor === undefined ? {} : { startedFor: child.startedFor }), + endedAt: session.journal.cursor() + } + return true +} + +function matchingChild( + session: ChildBearer, + identity: StructuredAgentSessionProviderChildIdentity +): StructuredAgentSessionProviderChild | null { + const { child } = session + return child && child.generation === identity.generation && child.fence === identity.fence + ? child + : null +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts index c8a2e4bba14f..04106c124fde 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts @@ -60,7 +60,7 @@ function createHost( async function abandonHost(host: StructuredAgentSessionHost): Promise { host['runtimeState'].stopLeaseRenewal() - host['holds'].dispose() + host['lifetime'].dispose() await Promise.all([...host['sessions'].values()].map((session) => session.journal.close())) host['sessions'].clear() } @@ -98,9 +98,9 @@ describe('structured session provider restore', () => { reason: 'read does not need ownership' })) - await restarted.restoreReadableSessions() + await restarted.restoreStartupSessions() - expect(restarted.listSessionTabs()).toEqual([ + expect(restarted.listPersistedSessionTabs([HOST_TEST_SESSION])).toEqual([ { sessionId: HOST_TEST_SESSION, workspaceId: 'workspace-1', agent: 'claude' } ]) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts index 52699fd062f6..7c195a9a1243 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts @@ -80,8 +80,8 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }) }) -function claudeParams() { - return hostTestAttachParams(null, { +function claudeParams(expectedRuntimeFence: number | null = null) { + return hostTestAttachParams(expectedRuntimeFence, { provider: 'claude', agent: 'claude', accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, @@ -102,7 +102,7 @@ describe('a publish-first Claude create whose init is slow', () => { expect(store.getRecord(SESSION)?.options?.model).toBeUndefined() expect(lastPhase()).toBe('starting') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('claude-opus-9') @@ -116,7 +116,7 @@ describe('a publish-first Claude create whose init is slow', () => { ).resolves.toMatchObject({ ok: true }) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') @@ -126,19 +126,21 @@ describe('a publish-first Claude create whose init is slow', () => { it('keeps the picked model across a resume whose new child starts on its own default', async () => { const params = claudeParams() await host.attach(CALLER, { ...params, options: { model: 'opus' } }) - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) await host.close(SESSION) const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 - // Reopening the chat: the surface's first hold resumes the session. - await host.hold(SESSION, 'chat-1') + // Starting the chat again resumes the session under a new fence. + await expect(host.attach(CALLER, claudeParams(releasedFence))).resolves.toMatchObject({ + ok: true + }) expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(releasedFence) // The new child's init reports its CLI default; the saved pick is restored over it. expect(store.getRecord(SESSION)?.options?.model).toBe('opus') expect(lastPhase()).toBe('starting') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts index 55c718f967b9..8f3c0390aaa1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -16,6 +16,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { nativeSessionOptionsFromReport } from './structured-agent-session-option-restoration' +import { markProviderChildStarted } from './structured-agent-session-provider-child' export type StructuredAgentSessionProviderStartedContext = { deps: StructuredAgentSessionHostDeps @@ -23,7 +24,6 @@ export type StructuredAgentSessionProviderStartedContext = { serialize: (sessionId: string, task: () => Promise) => Promise now: () => number publishStatus?: (sessionId: string) => void - restartReleaseGrace: (sessionId: string) => void onBarrierError: (sessionId: string, error: unknown) => void } @@ -35,16 +35,14 @@ export function settleStructuredAgentSessionProviderStarted( return context.serialize(event.sessionId, async () => { const session = context.sessions.get(event.sessionId) if ( - !session?.hasProviderChild || - session.fence !== event.fence || - session.acquisitionGeneration !== event.acquisitionGeneration + !session || + !markProviderChildStarted(session, { + generation: event.acquisitionGeneration, + fence: event.fence + }) ) { return } - session.providerChildPhase = 'ready' - // Prompts held for the start are written now but open a turn only on their echo; a release - // tick in between would stop the child before it runs them. - context.restartReleaseGrace(event.sessionId) try { await persistStartedOptions(context, event) } catch (error) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts deleted file mode 100644 index 34e3fb8a4cf8..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -// Read restore decides whether a session comes back at all. -// -// A chat still in the pre-SQLite format has no `journal.db`, so the probe that -// loads one reports nothing. Reading that as "no session" is what removed these -// chats: an unpublished session is also what prunes its tab out of the saved -// workspace, so the tab is gone before anything can explain itself. - -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { journalDirectoryFor } from '../agent-session-journal/journal-paths' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' - -const SESSION_ID = 'codex_read_restore_fixture' -const WORKSPACE_ID = 'repo-1::/tmp/workspace' - -const RECORD = { - schemaVersion: 2, - sessionId: SESSION_ID, - location: { - executionHostId: 'local', - wslDistro: null, - workspaceId: WORKSPACE_ID, - workspaceKind: 'git-worktree' - }, - provider: 'codex', - providerHandleChain: [ - { - linkId: 'codex-1-thread-1', - handle: { provider: 'codex', threadId: 'thread-1' }, - origin: 'created', - mintedAtFence: 1, - observedAt: 1 - } - ], - accountHome: { variable: 'CODEX_HOME', path: '/tmp/codex-home' }, - createdAt: 1, - updatedAt: 2, - lease: { sessionId: SESSION_ID, runtimeKind: 'native', runtimeFence: 1 } -} as unknown as AgentSessionRecord - -const store = { - getRecord: (sessionId: string) => (sessionId === SESSION_ID ? RECORD : null) -} as unknown as AgentSessionRecordStore - -let journalRoot: string -const opened: AgentSessionJournal[] = [] - -async function writeRemnant(name: string): Promise { - const dir = journalDirectoryFor(journalRoot, { - workspaceId: WORKSPACE_ID, - sessionId: SESSION_ID - }) - await mkdir(dir, { recursive: true }) - await writeFile(join(dir, name), '{"kind":"epoch","v":1,"seq":1}\n', 'utf8') - return join(dir, name) -} - -beforeEach(async () => { - journalRoot = await mkdtemp(join(tmpdir(), 'orca-read-restore-')) -}) - -afterEach(async () => { - await Promise.allSettled(opened.splice(0).map((journal) => journal.close())) - await rm(journalRoot, { recursive: true, force: true }) -}) - -describe('a session whose journal is still the pre-SQLite format', () => { - it('is published, carrying the message that explains it', async () => { - const transcript = await writeRemnant('log.jsonl') - - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) - - expect(restored).not.toBeNull() - opened.push(restored!.journal) - const disclosed = restored!.journal - .snapshot() - .items.map((entry) => (entry.body.kind === 'status' ? entry.body.text : '')) - expect(disclosed.join('')).toContain(transcript) - // Publishing it costs no agent process; acquisition still waits for the user. - expect(restored!.hasProviderChild).toBe(false) - }) - - it('is published for a remnant whose log is gone', async () => { - await writeRemnant('snapshot.json') - - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) - - expect(restored).not.toBeNull() - opened.push(restored!.journal) - }) - - it('still drops a session with neither a journal nor a remnant', async () => { - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) - - expect(restored).toBeNull() - }) - - it('still drops a session with no record', async () => { - await writeRemnant('log.jsonl') - - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, 'unknown-session') - - expect(restored).toBeNull() - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts deleted file mode 100644 index 65b913daf42a..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant' -import { loadJournal } from '../agent-session-journal/journal-open' -import { journalDirectoryFor } from '../agent-session-journal/journal-paths' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { - attachFingerprintFields, - journalIdentityFor, - type AgentSessionAttachParams -} from './structured-agent-session-attach' -import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' - -export type RestoredStructuredAgentSessionRead = { - journal: AgentSessionJournal - params: AgentSessionAttachParams - fence: number - hasProviderChild: false - providerChildPhase: 'ready' - acquisitionGeneration: null -} - -export async function restoreStructuredAgentSessionRead( - store: AgentSessionRecordStore, - journalRoot: string, - sessionId: string -): Promise { - const record = store.getRecord(sessionId) - if (!record) { - return null - } - const params = attachParamsForRecord(record, { - clientOperationId: `read-restore:${record.sessionId}`, - expectedRuntimeFence: record.lease.runtimeFence - }) - const journalDir = journalDirectoryFor(journalRoot, { - workspaceId: record.location.workspaceId, - sessionId - }) - const loaded = loadJournal(journalDir, sessionId) - if (loaded?.corrupt) { - return null - } - // A session still in the pre-SQLite format has no `journal.db` to load. Dropping - // it here leaves it unpublished, which is also what prunes its tab out of the - // saved workspace — so the chat disappears with nowhere to explain itself. - if (!loaded && !findJournalFileFormatRemnant(journalDir)) { - return null - } - const journal = await openAgentSessionJournal({ - identity: journalIdentityFor(record, params), - journalDir, - // Omitted, not `null`: the store reads `null` as "replay already ran and - // found nothing" and founds a fresh epoch. In process the probe above is the - // previous statement, so the window is zero-width; this holds the line for a - // database another process creates in between. - ...(loaded ? { loaded } : {}) - }) - // Read restore opens the journal and nothing else: no adapter call, so no - // provider child. Opening it can still write — a session whose history is in - // the old format founds its epoch and commits the row explaining that here. - return { - journal, - params, - fence: record.lease.runtimeFence, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null - } -} - -export function attachParamsForRecord( - record: AgentSessionRecord, - input: { - clientOperationId: string - expectedRuntimeFence: number - } -): AgentSessionAttachParams { - const params: AgentSessionAttachParams = { - envelope: { - sessionId: record.sessionId, - clientOperationId: input.clientOperationId, - expectedRuntimeFence: input.expectedRuntimeFence, - payloadFingerprint: '' - }, - location: record.location, - provider: record.provider, - agent: record.provider, - accountHome: record.accountHome, - runtimeKind: 'native' - } - return { - ...params, - envelope: { - ...params.envelope, - payloadFingerprint: computeAgentSessionPayloadFingerprint({ - method: 'agentSession.attach', - sessionId: record.sessionId, - fields: attachFingerprintFields(params) - }) - } - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts deleted file mode 100644 index 5d579fbe256e..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' - -const { restoreOnRestart } = vi.hoisted(() => ({ restoreOnRestart: vi.fn() })) - -vi.mock('./structured-agent-session-restart-restore', () => ({ - restoreStructuredAgentSessionsOnRestart: restoreOnRestart -})) - -import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' - -describe('StructuredAgentSessionReadableRestorer', () => { - beforeEach(() => { - restoreOnRestart.mockReset().mockResolvedValue(undefined) - }) - - it('passes targeted records to the restore pool in visible-first order', async () => { - const records = ['background-a', 'visible-b', 'visible-a', 'background-b'].map( - (sessionId) => ({ sessionId }) as AgentSessionRecord - ) - const restorer = new StructuredAgentSessionReadableRestorer({ - store: { listRecords: () => records } as never, - journalRoot: '/tmp/journals', - supportsRecord: () => true, - reconcile: async () => null, - resolveRecovery: async () => undefined, - serialize: async (_sessionId, task) => task(), - hasSession: () => false, - onReadable: () => undefined, - retrySettlement: async () => true - }) - - await restorer.restore(['visible-a', 'visible-b', 'background-a', 'background-b']) - - expect(restoreOnRestart).toHaveBeenCalledOnce() - expect(restoreOnRestart.mock.calls[0][0].records.map((record) => record.sessionId)).toEqual([ - 'visible-a', - 'visible-b', - 'background-a', - 'background-b' - ]) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts deleted file mode 100644 index a11c3a85b6c1..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import type { RestoredStructuredAgentSessionRead } from './structured-agent-session-read-restore' -import { - restoreOneStructuredAgentSessionRead, - restoreOneStructuredAgentSessionReadUnderSerialize, - restoreStructuredAgentSessionsOnRestart -} from './structured-agent-session-restart-restore' - -export class StructuredAgentSessionReadableRestorer { - private restorePromise: Promise | null = null - - constructor( - private readonly input: { - store: AgentSessionRecordStore - journalRoot: string - supportsRecord: (record: AgentSessionRecord) => boolean - reconcile: (sessionId: string) => Promise - resolveRecovery: (sessionId: string) => Promise - serialize: (sessionId: string, task: () => Promise) => Promise - hasSession: (sessionId: string) => boolean - onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void - retrySettlement: ( - sessionId: string, - params: RestoredStructuredAgentSessionRead['params'] - ) => Promise - } - ) {} - - restore(sessionIds?: readonly string[]): Promise { - this.restorePromise ??= this.restoreReadableSessions(sessionIds).catch((error: unknown) => { - this.restorePromise = null - throw error - }) - return this.restorePromise - } - - /** - * One session, on demand, after startup. - * - * Deliberately outside `restorePromise`: that latch answers "has the startup sweep run", and a - * surface asking for a session the sweep never covered — or that was closed since — must not be - * told yes because the sweep finished. Needs no dedupe of its own; the per-session task queue - * `serialize` runs on already orders concurrent callers, and the second one sees `hasSession`. - * - * Provider-agnostic by construction: eligibility is `supportsRecord`, which the adapter router - * answers for Claude and Codex from the record's own provider. - */ - async restoreOne(sessionId: string): Promise { - if (!this.supports(sessionId)) { - return false - } - await restoreOneStructuredAgentSessionRead(this.input, sessionId) - return this.input.hasSession(sessionId) - } - - /** `restoreOne` for a caller already inside the session's serialize. Reconciliation is skipped - * on purpose: a lease this host has not adjudicated is the attach's problem, and a replay - * needs only the journal. */ - async restoreOneUnderSerialize(sessionId: string): Promise { - if (!this.supports(sessionId)) { - return false - } - await restoreOneStructuredAgentSessionReadUnderSerialize(this.input, sessionId) - return this.input.hasSession(sessionId) - } - - private supports(sessionId: string): boolean { - const record = this.input.store.getRecord(sessionId) - return record !== null && this.input.supportsRecord(record) - } - - private async restoreReadableSessions(sessionIds?: readonly string[]): Promise { - const targetOrder = sessionIds - ? new Map(sessionIds.map((sessionId, index) => [sessionId, index])) - : null - const records = this.input.store - .listRecords() - .filter( - (record) => - this.input.supportsRecord(record) && (!targetOrder || targetOrder.has(record.sessionId)) - ) - if (targetOrder) { - records.sort( - (left, right) => targetOrder.get(left.sessionId)! - targetOrder.get(right.sessionId)! - ) - } - await restoreStructuredAgentSessionsOnRestart({ - ...this.input, - records - }) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-exits.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-exits.test.ts index 7cd92bf521bf..6e3dd56886ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-exits.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-exits.test.ts @@ -59,6 +59,12 @@ async function stopOwner(child: ReturnType): Promise spawnedOwners.delete(child) } +/** What a send's delivery or `agentSession.ensure` does: attach at the record's current fence. */ +async function startAgent(): Promise { + const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? null + expect((await host.attach(CALLER, hostTestAttachParams(fence))).ok).toBe(true) +} + function adapter(): StructuredAgentSessionAdapter { return { acquire, @@ -84,7 +90,7 @@ function openHost(overrides: Partial = {}): void async function abandonHost(abandonedHost: StructuredAgentSessionHost): Promise { abandonedHost['runtimeState'].stopLeaseRenewal() - abandonedHost['holds'].dispose() + abandonedHost['lifetime'].dispose() await Promise.all( [...abandonedHost['sessions'].values()].map((session) => session.journal.close()) ) @@ -183,7 +189,7 @@ describe('recovery exits', () => { mintSpawnToken: () => 'spawn-c', probeOwner: async () => ({ outcome: 'pid-absent' }) }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() // Death proof releases the residue outright: no handoff continuation, no spawned child, // no stage a user would have to clear by hand. @@ -197,8 +203,8 @@ describe('recovery exits', () => { runtimeFence: 4 }) - // The ordinary native recovery path remains: the first surface hold resumes it. - await host.hold(SESSION, 'surface-1') + // The ordinary native recovery path remains: the next start resumes it. + await startAgent() expect(acquire).toHaveBeenCalledTimes(3) expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live', @@ -242,7 +248,7 @@ describe('recovery exits', () => { }) }) - it('heals a stranded native owner during startup restore, and spawns nothing until a surface asks', async () => { + it('heals a stranded native owner during startup restore, and spawns nothing until work asks', async () => { expect((await host.attach(CALLER, hostTestAttachParams(null))).ok).toBe(true) await reopenStore() @@ -259,10 +265,10 @@ describe('recovery exits', () => { stopOwnerProcess }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() // Healing is startup's job; spawning is not. The orphan is stopped and the lease is free, but - // nothing has asked to look at this session, so no replacement child exists yet. + // nothing has asked this session for work, so no replacement child exists yet. expect(stopOwnerProcess).toHaveBeenCalledWith(4242, 'SIGTERM') expect(acquire).toHaveBeenCalledTimes(1) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -271,7 +277,7 @@ describe('recovery exits', () => { ownerProcess: null }) - await host.hold(SESSION, 'surface-1') + await startAgent() expect(acquire).toHaveBeenCalledTimes(2) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -320,7 +326,7 @@ describe('recovery exits', () => { } }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeFence: 2, claimStatus: 'released', @@ -340,7 +346,7 @@ describe('recovery exits', () => { } }) - await host.hold(SESSION, 'surface-overlap') + await startAgent() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeFence: 3, @@ -348,6 +354,6 @@ describe('recovery exits', () => { handoffStage: null, ownerProcess: { pid: replacement.process.pid, spawnToken: 'spawn-b' } }) - expect(host.history({ sessionId: SESSION, direction: 'tail' }).ok).toBe(true) + expect((await host.history({ sessionId: SESSION, direction: 'tail' })).ok).toBe(true) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts index 87f4f9e3d65e..8f686aeab7a8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts @@ -105,7 +105,8 @@ async function createHarness(options: { attached?: boolean } = {}) { async function abandonHost(host: StructuredAgentSessionHost): Promise { host['runtimeState'].stopLeaseRenewal() - host['holds'].dispose() + host['lifetime'].dispose() + host['conversationDelivery'].loop.dispose() await Promise.all([...host['sessions'].values()].map((session) => session.journal.close())) host['sessions'].clear() } @@ -225,16 +226,20 @@ const UNREACHABLE = new Set([ // StructuredAgentSessionHost.mutate maps an absent record to AGENT_SESSION_NOT_ATTACHED. 'agentSession.setOption:agent_session_identity_required', 'agentSession.send:agent_session_identity_required', - // No structured-agent-session host branch emits agent_session_journal_unreadable. + // Only a send opens the conversation it writes to. 'agentSession.setOption:agent_session_journal_unreadable', - 'agentSession.send:agent_session_journal_unreadable', // Send reconstructs doubt from its global tombstone instead of refusing it. 'agentSession.send:agent_session_operation_unknown', // Only a send restarts a lost owner. 'agentSession.setOption:agent_session_owner_restart_failed', // A write names its target, not an owner generation; only an attach compares fences. 'agentSession.setOption:agent_session_checkpoint_stale', - 'agentSession.send:agent_session_checkpoint_stale' + 'agentSession.send:agent_session_checkpoint_stale', + // A send is a conversation write: admitted whoever owns the lease, and a start it needs that + // fails rejects the accepted message rather than refusing the call. + 'agentSession.send:agent_session_conflict', + 'agentSession.send:execution_owner_reconciling', + 'agentSession.send:agent_session_owner_restart_failed' ]) describe('agentSessionRefusalOperationState host oracle', () => { @@ -251,7 +256,7 @@ describe('agentSessionRefusalOperationState host oracle', () => { expect(stale.setOption).toHaveBeenCalledTimes(1) const conflict = await createHarness() - for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + for (const method of ['agentSession.setOption'] as const) { await setLease(conflict, (current) => ({ ...current, lease: { ...current.lease, handoffStage: 'new-owner-proving' } @@ -331,7 +336,7 @@ describe('agentSessionRefusalOperationState host oracle', () => { record(await assertHostAgreement(unknown, optionUnknown, 'agent_session_operation_unknown')) const reconciling = await createHarness() - for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + for (const method of ['agentSession.setOption'] as const) { await setLease(reconciling, (current) => ({ ...current, lease: { ...current.lease, unreconciled: true } @@ -348,16 +353,21 @@ describe('agentSessionRefusalOperationState host oracle', () => { ) } - const unrecoverable = await createHarness() - await unrecoverable.host.close(SESSION) - unrecoverable.host.deps.adapter.acquire = async () => { - throw new Error('no provider thread to resume') + const unreadable = await createHarness() + await unreadable.host.close(SESSION) + unreadable.host.deps.adapter.historyFilePath = async () => { + throw new Error('transcript unreadable') } + const unreadableSend = { method: 'agentSession.send' as const, operationId: operationId() } record( await assertHostAgreement( - unrecoverable, - { method: 'agentSession.send', operationId: operationId() }, - 'agent_session_owner_restart_failed' + unreadable, + unreadableSend, + 'agent_session_journal_unreadable', + async () => { + delete unreadable.host.deps.adapter.historyFilePath + return { harness: unreadable, spec: unreadableSend } + } ) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-release-clock.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-release-clock.ts deleted file mode 100644 index 91daa5606a81..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-release-clock.ts +++ /dev/null @@ -1,85 +0,0 @@ -// The delay between "nothing holds this session and nothing has happened in it" and "stop its -// provider child". -// -// It is an IDLE window, not a short grace. A surface that reconnects — a mobile socket dropping on -// a network switch, a renderer remounting a tab, a worktree switch hiding the pane — releases and -// re-holds, and a send to a chat nobody is looking at restarts its owner; stopping the child soon -// after either costs the user a respawn plus a resume on the next message. And a turn the user -// already asked for must finish: stopping the child mid-answer strands the open turn marker. -// -// So the clock arms when the last holder leaves, every journal write while it is armed starts it -// again, and a tick that finds work still owed — a turn running, a message sent but not yet -// taken by the provider, or a subagent, command or monitor still running — re-arms instead of -// evicting. The child goes only after a full window -// with no holder and no owed work. Quit still stops every child at once. - -export const STRUCTURED_AGENT_SESSION_RELEASE_GRACE_MS = 30 * 60_000 - -export type StructuredAgentSessionReleaseClockDeps = { - /** Never evict while work is owed; a true answer re-arms the clock instead. */ - hasOwedWork: (sessionId: string) => boolean - /** Re-checked at fire time: a holder may have arrived while the timer ran. */ - isHeld: (sessionId: string) => boolean - evict: (sessionId: string) => Promise - onError?: (input: { sessionId: string; error: unknown }) => void - graceMs?: number -} - -export class StructuredAgentSessionReleaseClock { - private readonly timers = new Map>() - private readonly graceMs: number - - constructor(private readonly deps: StructuredAgentSessionReleaseClockDeps) { - this.graceMs = deps.graceMs ?? STRUCTURED_AGENT_SESSION_RELEASE_GRACE_MS - } - - arm(sessionId: string): void { - this.cancel(sessionId) - const timer = setTimeout(() => { - this.timers.delete(sessionId) - this.fire(sessionId) - }, this.graceMs) - // A pending release must never be the reason a process stays alive at quit. - timer.unref?.() - this.timers.set(sessionId, timer) - } - - /** Activity in an unheld session: the idle window starts over. */ - renew(sessionId: string): void { - if (this.timers.has(sessionId)) { - this.arm(sessionId) - } - } - - cancel(sessionId: string): void { - const timer = this.timers.get(sessionId) - if (timer) { - clearTimeout(timer) - this.timers.delete(sessionId) - } - } - - isArmed(sessionId: string): boolean { - return this.timers.has(sessionId) - } - - dispose(): void { - for (const timer of this.timers.values()) { - clearTimeout(timer) - } - this.timers.clear() - } - - private fire(sessionId: string): void { - if (this.deps.isHeld(sessionId)) { - return - } - if (this.deps.hasOwedWork(sessionId)) { - this.arm(sessionId) - return - } - void this.deps.evict(sessionId).catch((error: unknown) => { - this.deps.onError?.({ sessionId, error }) - }) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts new file mode 100644 index 000000000000..3ea116fa361d --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts @@ -0,0 +1,224 @@ +// A real host over a real store and journal, with a scripted provider and a clock the test moves, +// for the tests of a conversation that outlives its agent. The idle sweep runs on its own short +// interval; a test moves `clock.now` past the idle window and waits for the outcome. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import type { + AgentSessionMutationEnvelope, + AgentSessionStatusEvent, + AgentSessionSubscribeEvent +} from '../../../shared/agent-session-wire' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { + AgentSessionDispatchOutcome, + StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { + HOST_TEST_NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' +import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idle-sweep' + +export const REST_TEST_CALLER = { callerKey: 'client-1' } +export const IDLE_MS = STRUCTURED_AGENT_SESSION_IDLE_MS +export const SWEEP_INTERVAL_MS = 5 + +export type RestTestAdapter = { + acquire: Mock + closeSession: Mock> + dispatch: Mock + acknowledgeSessionRelease: Mock< + NonNullable + > + backgroundTaskState: Mock> + readOptions: Mock> +} + +export type RestTestRig = { + root: string + store: AgentSessionRecordStore + host: StructuredAgentSessionHost + adapter: RestTestAdapter + clock: { now: number } + statusEvents: AgentSessionStatusEvent[] + sink: { publish: Mock; forget: Mock } + /** Opens a fresh host over the same store and journals: what a restart leaves behind. */ + restart: (deps?: Partial) => Promise + dispose: () => Promise +} + +let ordinal = 0 +let generations = 0 + +export function acceptedDispatch(): AgentSessionDispatchOutcome { + ordinal += 1 + return { + state: 'accepted', + providerIdentity: { provider: 'codex', threadId: THREAD, turnId: `turn-${ordinal}`, ordinal } + } +} + +export function restTestSend( + text: string, + fence = 1 +): { envelope: AgentSessionMutationEnvelope; body: AgentJournalMessageItem } { + const body = hostTestMessage(text) + return { + body, + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + } + } +} + +/** Every item and submission a reader was sent, whatever frame carried it. */ +export function readerSaw(events: readonly AgentSessionSubscribeEvent[]) { + const items = events.flatMap((event) => + event.type === 'batch' ? event.batch.items : event.type === 'end' ? [] : event.page.items + ) + const submissions = events.flatMap((event) => + event.type === 'batch' ? event.batch.submissions : [] + ) + return { + texts: items.flatMap((item) => + item.body.kind === 'message' + ? item.body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])) + : [] + ), + submissions + } +} + +export function collectSubscriber(): { + events: AgentSessionSubscribeEvent[] + emit: (event: AgentSessionSubscribeEvent) => void +} { + const events: AgentSessionSubscribeEvent[] = [] + return { events, emit: (event) => events.push(event) } +} + +export async function createRestTestRig( + deps: Partial = {} +): Promise { + resetHostTestOperationIds() + ordinal = 0 + const root = await mkdtemp(join(tmpdir(), 'orca-rest-')) + const clock = { now: HOST_TEST_NOW } + const statusEvents: AgentSessionStatusEvent[] = [] + const sink = { publish: vi.fn(), forget: vi.fn() } + let store = await AgentSessionRecordStore.open({ + directory: join(root, 'store'), + hostId: 'local' + }) + const adapter: RestTestAdapter = { + acquire: vi.fn(async ({ fence, spawnToken }) => ({ + acquisitionGeneration: `generation-${++generations}`, + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: clock.now + } + })), + closeSession: vi.fn(async () => true), + dispatch: vi.fn(async () => acceptedDispatch()), + acknowledgeSessionRelease: vi.fn(), + backgroundTaskState: vi.fn(() => undefined), + readOptions: vi.fn(async () => ({ models: [], current: { model: 'gpt-live' } })) + } + const hostFor = (overrides: Partial) => + new StructuredAgentSessionHost({ + store, + adapter: { + ...adapter, + releaseAcquisition: vi.fn(async () => true), + cancelTurn: async () => ({ cancelled: true }), + answerPrompt: async ({ commit }) => commit(), + setOption: async () => undefined + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + probeOwner: async () => ({ outcome: 'pid-absent' }), + now: () => clock.now, + statusSink: sink, + idleSweep: { intervalMs: SWEEP_INTERVAL_MS }, + ...deps, + ...overrides + }) + const rig: RestTestRig = { + root, + store, + host: hostFor({}), + adapter, + clock, + statusEvents, + sink, + restart: async (overrides = {}) => { + await rig.host.flushAllStreamedEvents().catch(() => undefined) + store = await AgentSessionRecordStore.open({ + directory: join(root, 'store'), + hostId: 'local' + }) + rig.store = store + rig.host = hostFor(overrides) + rig.host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) + return rig.host + }, + dispose: async () => { + await rig.host.flushAllStreamedEvents().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + } + } + rig.host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) + return rig +} + +/** Creates the chat, lists its tab, and sends one message so its journal is on disk. */ +export async function foundRestTestChat(rig: RestTestRig): Promise { + const attached = await rig.host.attach(REST_TEST_CALLER, hostTestAttachParams(null)) + if (!attached.ok) { + throw new Error(`attach refused: ${attached.refusal.code}`) + } + await rig.store.setSessionTabVisibility(SESSION, true) + const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('hello', attached.fence)) + if (!sent.ok) { + throw new Error(`send refused: ${sent.refusal.code}`) + } + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalled()) +} + +/** Runs one sweep pass now, for a test that set `idleSweep.intervalMs` out of reach. */ +export function sweepOnce(host: StructuredAgentSessionHost): Promise { + return host.collaboratorsForTests().lifetime.idleSweep.tick() +} + +/** Waits long enough for several sweep ticks to have run. */ +export function sweepTicks(count = 6): Promise { + return new Promise((resolve) => setTimeout(resolve, SWEEP_INTERVAL_MS * count)) +} + +export { SESSION as REST_TEST_SESSION, THREAD as REST_TEST_THREAD } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-candidates.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-candidates.ts index 46879aefe49a..50008ce79c44 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-candidates.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-candidates.ts @@ -2,16 +2,11 @@ // // A different question from storage: the durable record decides which markers are still present; // this decides which of those a resume may act on. The offer, the click and the pre-send check all -// ask it, and all get the same answer: the marker stands until the user sends a newer message. +// ask it, and all get the same answer. -import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' -import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' -import { - latestStructuredAgentSessionPrompt, - latestStructuredAgentSessionUserItem -} from '../../../shared/structured-agent-session-projection' +import { latestStructuredAgentSessionPrompt } from '../../../shared/structured-agent-session-projection' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { adapterSupportsRecord } from './structured-agent-session-provider-support' @@ -23,61 +18,29 @@ import { /** The only part of a live session this reads. */ export type StructuredAgentSessionRestartJournalSource = { journal: AgentSessionJournal } -export type StructuredAgentSessionRestartCandidateOptions = { - /** A continuation already in flight; its own submission is not newer user work. */ - pendingContinuationId?: string -} - export type StructuredAgentSessionRestartCandidateReader = ( markers: readonly AgentSessionResumeMarker[], - leaseState: 'must-be-released' | 'may-be-held', - options?: StructuredAgentSessionRestartCandidateOptions + leaseState: 'must-be-released' | 'may-be-held' ) => StructuredAgentSessionResumableSet -/** The newest user message in a live session's journal, the same fact the predicate compares - * against a marker. Undefined when the session is not readable here, which decides nothing. */ -export function liveStructuredAgentSessionLatestUserItemId( - sessions: ReadonlyMap, - sessionId: string -): string | null | undefined { - const session = sessions.get(sessionId) - return session - ? (latestStructuredAgentSessionUserItem(session.journal.snapshot().items)?.itemId ?? null) - : undefined -} - export function createStructuredAgentSessionRestartCandidateReader(deps: { /** The host's live session map; a marker's chat is readable once listing has revealed it. */ sessions: ReadonlyMap getRecord: (sessionId: string) => AgentSessionRecord | null adapter: StructuredAgentSessionAdapter + /** Whether the chat moved on since the offer was taken; see the offer withdrawal. */ + movedOn: (marker: AgentSessionResumeMarker) => boolean }): StructuredAgentSessionRestartCandidateReader { - return (markers, leaseState, options = {}) => { - const items = new Map() - const itemsFor = (sessionId: string): AgentJournalRenderItem[] | undefined => { - if (items.has(sessionId)) { - return items.get(sessionId) - } - let snapshot = deps.sessions.get(sessionId)?.journal.snapshot().items - if (snapshot && options.pendingContinuationId) { - const ownItemId = agentJournalSubmissionKey(options.pendingContinuationId) - snapshot = snapshot.filter((item) => item.itemId !== ownItemId) - } - items.set(sessionId, snapshot) - return snapshot - } - return structuredAgentSessionResumableSet({ + return (markers, leaseState) => + structuredAgentSessionResumableSet({ markers, getRecord: deps.getRecord, supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), - latestPrompt: (sessionId) => latestStructuredAgentSessionPrompt(itemsFor(sessionId) ?? []), - latestUserItemId: (sessionId) => { - const snapshot = itemsFor(sessionId) - return snapshot === undefined - ? undefined - : (latestStructuredAgentSessionUserItem(snapshot)?.itemId ?? null) - }, + movedOn: deps.movedOn, + latestPrompt: (sessionId) => + latestStructuredAgentSessionPrompt( + deps.sessions.get(sessionId)?.journal.snapshot().items ?? [] + ), leaseState }) - } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-cap.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-cap.test.ts new file mode 100644 index 000000000000..9ded5b54d983 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-cap.test.ts @@ -0,0 +1,86 @@ +// "Resume all" starts agents a few at a time. A continuation is a send, so a chat counts as started +// once its agent took the message or its start failed — not when the message was accepted, which +// is instant and would let every agent start at once. + +import { expect, it, vi } from 'vitest' +import { + startStructuredAgentSessionContinuation, + type StructuredAgentSessionContinuationDeps +} from './structured-agent-session-restart-continuation' +import { + resumeStructuredAgentSessionsFromRestart, + STRUCTURED_AGENT_SESSION_RESUME_CONCURRENCY, + StructuredAgentSessionResumeAdmission +} from './structured-agent-session-restart-resume-runner' +import { marker } from './structured-agent-session-restart-resume-test-harness' + +type Handover = { dispatchState: string; reason?: string | null } | undefined + +it('holds each slot from accept until handover or rejection, and frees it when the wait ends (P2-27)', async () => { + const handovers = new Map>() + let waiting = 0 + let mostWaiting = 0 + const deps = (sessionId: string): StructuredAgentSessionContinuationDeps => ({ + currentFence: () => 1, + send: vi.fn(async () => ({ ok: true, value: { submission: { dispatchState: 'pending' } } })), + awaitHandedOver: async () => { + const handover = Promise.withResolvers() + handovers.set(sessionId, handover) + waiting += 1 + mostWaiting = Math.max(mostWaiting, waiting) + try { + return await handover.promise + } finally { + waiting -= 1 + } + }, + awaitSettlement: async () => ({ dispatchState: 'accepted' }), + note: async () => undefined, + onNoteFailed: () => undefined + }) + const sessions = Array.from({ length: 8 }, (_, index) => `session-${index}`) + const outcomes = resumeStructuredAgentSessionsFromRestart( + { + admission: new StructuredAgentSessionResumeAdmission(), + consumeMarker: async () => true, + resume: async (sessionId) => { + const started = await startStructuredAgentSessionContinuation( + deps(sessionId), + sessionId, + marker(), + 'operation-1' + ) + if ('done' in started && started.done.outcome === 'refused') { + throw new Error(started.done.reason ?? 'refused') + } + } + }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the runner reads only each candidate's session id. + sessions.map((sessionId) => ({ sessionId }) as never), + 'modal' + ) + const settle = async (sessionId: string, handover: Handover) => { + await vi.waitFor(() => expect(handovers.has(sessionId)).toBe(true)) + handovers.get(sessionId)!.resolve(handover) + } + + await vi.waitFor(() => expect(waiting).toBe(STRUCTURED_AGENT_SESSION_RESUME_CONCURRENCY)) + // Accepted but not yet handed over: a slow start keeps its slot. + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(handovers.size).toBe(STRUCTURED_AGENT_SESSION_RESUME_CONCURRENCY) + await settle('session-0', { dispatchState: 'pending' }) + await settle('session-1', { dispatchState: 'rejected', reason: 'Codex could not start.' }) + // The session closed, or too many waited: the wait proves nothing, and the slot is freed. + await settle('session-2', undefined) + for (const sessionId of sessions.slice(3)) { + await settle(sessionId, { dispatchState: 'pending' }) + } + + const results = await outcomes + expect(mostWaiting).toBe(STRUCTURED_AGENT_SESSION_RESUME_CONCURRENCY) + expect(results.find((result) => result.sessionId === 'session-1')).toMatchObject({ + outcome: 'refused', + reason: 'Codex could not start.' + }) + expect(results.filter((result) => result.outcome === 'resumed')).toHaveLength(7) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-child-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-child-work.test.ts index 63c778144eef..6d8514c830e9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-child-work.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-child-work.test.ts @@ -2,10 +2,7 @@ import { expect, it, vi } from 'vitest' import { interruptedRestart } from './structured-agent-session-restart-interruption-test-harness' -import { - HOST_TEST_SESSION as SESSION, - HOST_TEST_THREAD as THREAD -} from './structured-agent-session-host-test-data' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' // What the user hit: the lead had finished, its subagent had not. The offer names the subagent // from the roster snapshot taken BEFORE the child stopped — the close that follows settles the @@ -21,65 +18,22 @@ it('offers a chat whose subagent the restart stopped, named from the stop-time s ]) }) -// On resume the provider restates what it lost in its own words, rewriting the row before the -// continuation is dispatched. What the offer was admitted for still stands. -it('continues a stopped subagent after the provider restates its row', async () => { - const { host, acquire, dispatch } = await interruptedRestart('children') - await host.hold(SESSION, 'pane') - const events = acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') - } - const admit = vi.spyOn(host.deps.store, 'admitMutationOperation') - admit.mockImplementationOnce(async (input) => { - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'settled-turn', ordinal: 2 }, - { - kind: 'message', - role: 'system', - blocks: [ - { - type: 'subagent-group', - groupId: 'settled-turn', - agents: [{ id: 'child-1', label: 'Review loop 4', state: 'completed' }] - } - ] - } - ) - await host.flushStreamedEvents(SESSION) - admit.mockRestore() - return host.deps.store.admitMutationOperation(input) - }) +// The offer was admitted for the subagent's work; the continuation asks the agent to carry it on. +it('continues a chat whose subagent the restart stopped', async () => { + const { host, dispatch } = await interruptedRestart('children') expect( (await host.restartResume.continueAfterRestart([SESSION], 'modal')).continued ).toMatchObject([{ outcome: 'continued' }]) expect(dispatch).toHaveBeenCalledTimes(1) - host.release(SESSION, 'pane') }) -// The user opens the chat before choosing Resume, and the reattached provider restates its row. +// The user opens the chat before choosing Resume: reading it starts nothing, so the offer stands. it('still offers and continues a chat the user opened before resuming', async () => { const { host, acquire, dispatch } = await interruptedRestart('children') - await host.hold(SESSION, 'pane') - const events = acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') - } - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'settled-turn', ordinal: 2 }, - { - kind: 'message', - role: 'system', - blocks: [ - { - type: 'subagent-group', - groupId: 'settled-turn', - agents: [{ id: 'child-1', label: 'Review loop 4', state: 'completed' }] - } - ] - } - ) - await host.flushStreamedEvents(SESSION) + const unsubscribe = await host.subscribe({ id: 'pane', sessionId: SESSION, emit: vi.fn() }) + await host.history({ sessionId: SESSION, direction: 'tail' }) + unsubscribe() + expect(acquire).not.toHaveBeenCalled() expect(await host.restartResume.list()).toMatchObject([ { sessionId: SESSION, activity: { tasks: [{ kind: 'agent', label: 'Review loop 4' }] } } ]) @@ -87,5 +41,4 @@ it('still offers and continues a chat the user opened before resuming', async () (await host.restartResume.continueAfterRestart([SESSION], 'modal')).continued ).toMatchObject([{ outcome: 'continued' }]) expect(dispatch).toHaveBeenCalledTimes(1) - host.release(SESSION, 'pane') }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-envelope.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-envelope.ts new file mode 100644 index 000000000000..cfa2c7b502ee --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-envelope.ts @@ -0,0 +1,77 @@ +// The restart continuation as a message: its body, which depends on the marker alone, and its +// identity, which is one per resume action. + +import { createHash } from 'node:crypto' +import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { restartContinuationMessage } from '../../../shared/agent-session-restart-continuation' +import type { AgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' +import type { AgentSessionMutationEnvelope } from '../../../shared/agent-session-wire' + +/** The message body, built once so both the send and any test read the same text. */ +export function restartContinuationBody(marker: AgentSessionResumeMarker): AgentJournalMessageItem { + return { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: restartContinuationMessage(marker) }] + } +} + +const hex16 = (parts: readonly unknown[]): string => + createHash('sha256').update(JSON.stringify(parts)).digest('hex').slice(0, 16) + +/** Names the offer: every continuation any of its actions sends carries it, and nothing else does. */ +function restartOfferTag(marker: AgentSessionResumeMarker): string { + return hex16([marker.teardownId, marker.sessionId]) +} + +/** One resume action's continuation: the offer's tag, then the action's own part, so a retry is a + * new message, never a replay of the one that failed. Dated by the action, not the quit: the + * ledger refuses a new id dated more than a day back, and an offer has no expiry. */ +export function restartContinuationId( + marker: AgentSessionResumeMarker, + operationId: string, + actionAt: number +): string { + return `${Math.trunc(actionAt).toString().padStart(13, '0')}-${restartOfferTag(marker)}${hex16([ + marker.teardownId, + marker.sessionId, + marker.work.kind, + marker.work.id, + marker.providerHandleRoot, + operationId + ])}` +} + +/** Whether a message is a continuation one of this offer's resume actions sent, read off its id. */ +export function isRestartContinuationOf( + marker: AgentSessionResumeMarker, + clientMessageId: string +): boolean { + return /^\d{13}-[0-9a-f]{32}$/.test(clientMessageId) + ? clientMessageId.slice(14, 30) === restartOfferTag(marker) + : false +} + +/** The fence only fills the envelope: admission names this send by its operation id, not a fence. */ +export function restartContinuationEnvelope( + sessionId: string, + fence: number, + marker: AgentSessionResumeMarker, + continuationId: string +): { envelope: AgentSessionMutationEnvelope; body: AgentJournalMessageItem } { + const body = restartContinuationBody(marker) + return { + body, + envelope: { + sessionId, + clientOperationId: continuationId, + expectedRuntimeFence: fence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId, + fields: { body } + }) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts new file mode 100644 index 000000000000..cfbdd9ce35e1 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts @@ -0,0 +1,112 @@ +// The restart continuation is accepted like any send, so its verdict is its delivery: a cold start +// longer than the legacy client wait must not turn a continuation that went through into an +// "unconfirmed" one. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' +import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' + +const SESSION = 'session-1' +const MESSAGE = 'continuation-1' + +let submission: AgentJournalSubmission +const journal = { + submissions: (): AgentJournalSubmission[] => [submission], + cursor: () => ({ epoch: 'epoch-1', sequence: 1 }) +} + +beforeEach(() => { + vi.useFakeTimers() + submission = { + clientMessageId: MESSAGE, + fence: 2, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending', + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true + } +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('the restart continuation waits for its delivery (W18)', () => { + it('reaches accepted after a 40 s cold start, with nothing filed as unconfirmed', async () => { + const settlement = new StructuredAgentSessionSendSettlement(() => journal) + const surfaces = structuredAgentSessionRestartResumeSurfaces( + { + revealSession: async () => ({ readable: true }), + send: async () => { + throw new Error('not used') + }, + waitForSendSettlement: settlement.wait + }, + () => 0 + ) + + const verdict = surfaces.awaitSendSettlement(SESSION, MESSAGE) + await vi.advanceTimersByTimeAsync(40_000) + submission = { ...submission, dispatchState: 'accepted', providerItemId: 'item-1' } + settlement.publish(SESSION, journal) + + await expect(verdict).resolves.toMatchObject({ + value: { submission: { dispatchState: 'accepted' } } + }) + }) +}) + +describe('a restart batch holds a chat until its continuation is handed over', () => { + function surfacesOver(settlement: StructuredAgentSessionSendSettlement) { + return structuredAgentSessionRestartResumeSurfaces( + { + revealSession: async () => ({ readable: true }), + send: async () => { + throw new Error('not used') + }, + waitForSendSettlement: settlement.wait + }, + () => 0 + ) + } + + it('keeps waiting through a slow start and resolves once the agent takes the message', async () => { + const settlement = new StructuredAgentSessionSendSettlement(() => journal) + let released = false + const handedOver = surfacesOver(settlement) + .awaitSendHandedOver(SESSION, MESSAGE) + .then((result) => { + released = true + return result + }) + await vi.advanceTimersByTimeAsync(5 * 60_000) + expect(released).toBe(false) + // Handed over: still pending at the provider, no longer queued. + submission = { ...submission, handedOverAt: 2 } + settlement.publish(SESSION, journal) + await expect(handedOver).resolves.toMatchObject({ + value: { submission: { dispatchState: 'pending', handedOverAt: 2 } } + }) + }) + + it('resolves once the start fails and the message is rejected', async () => { + const settlement = new StructuredAgentSessionSendSettlement(() => journal) + const handedOver = surfacesOver(settlement).awaitSendHandedOver(SESSION, MESSAGE) + submission = { ...submission, dispatchState: 'rejected', reason: 'Codex could not start.' } + settlement.publish(SESSION, journal) + await expect(handedOver).resolves.toMatchObject({ + value: { submission: { dispatchState: 'rejected' } } + }) + }) + + it('resolves undefined when the session closes first', async () => { + const settlement = new StructuredAgentSessionSendSettlement(() => journal) + const handedOver = surfacesOver(settlement).awaitSendHandedOver(SESSION, MESSAGE) + settlement.closeSession(SESSION) + await expect(handedOver).resolves.toBeUndefined() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.test.ts index 2d558f8f0287..0fe36994cd7c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.test.ts @@ -6,13 +6,22 @@ import { AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE } from '../../../shared/agent-session-restart-continuation' import { - continueStructuredAgentSessionAfterRestart, RestartContinuationSupersededError, + startStructuredAgentSessionContinuation, type StructuredAgentSessionContinuationDeps } from './structured-agent-session-restart-continuation' +/** The whole continuation: handed over, then its verdict. */ +async function continueStructuredAgentSessionAfterRestart( + ...args: Parameters +) { + const started = await startStructuredAgentSessionContinuation(...args) + return 'done' in started ? started.done : started.verdict() +} + function dependencies( - settledDispatch: 'accepted' | 'pending' | 'unknown' | 'rejected' + settledDispatch: 'accepted' | 'pending' | 'unknown' | 'rejected', + handedOver: 'pending' | 'rejected' | undefined = 'pending' ): StructuredAgentSessionContinuationDeps & { note: ReturnType send: ReturnType @@ -27,6 +36,14 @@ function dependencies( dispatchState: settledDispatch, reason: settledDispatch === 'rejected' ? 'provider_refused' : null })), + awaitHandedOver: vi.fn(async () => + handedOver + ? { + dispatchState: handedOver, + reason: handedOver === 'rejected' ? 'Codex could not start.' : null + } + : undefined + ), note: vi.fn(async () => undefined), onNoteFailed: vi.fn() } @@ -36,7 +53,7 @@ it('reports an accepted continuation and records its note', async () => { const deps = dependencies('accepted') await expect( - continueStructuredAgentSessionAfterRestart(deps, SESSION, marker()) + continueStructuredAgentSessionAfterRestart(deps, SESSION, marker(), 'operation-1') ).resolves.toEqual({ sessionId: SESSION, outcome: 'continued' @@ -58,20 +75,20 @@ it.each([ const deps = dependencies(settled) await expect( - continueStructuredAgentSessionAfterRestart(deps, SESSION, marker()) + continueStructuredAgentSessionAfterRestart(deps, SESSION, marker(), 'operation-1') ).resolves.toEqual(expected) expect(deps.note).toHaveBeenCalledExactlyOnceWith(...note) } ) -it('notes a superseded continuation in the chat and still reports the refusal', async () => { +it("writes nothing in the chat when the user's own message came first, and still refuses", async () => { const deps = dependencies('accepted') deps.send.mockRejectedValue(new RestartContinuationSupersededError()) await expect( - continueStructuredAgentSessionAfterRestart(deps, SESSION, marker()) + continueStructuredAgentSessionAfterRestart(deps, SESSION, marker(), 'operation-1') ).rejects.toBeInstanceOf(RestartContinuationSupersededError) - expect(deps.note).toHaveBeenCalledExactlyOnceWith(...REFUSED) + expect(deps.note).not.toHaveBeenCalled() }) // An ownership refusal would meet the user's own message too, so the note gives no advice to send one. @@ -83,7 +100,7 @@ it.each([ deps.send.mockResolvedValue({ ok: false, refusal: { code } }) await expect( - continueStructuredAgentSessionAfterRestart(deps, SESSION, marker()) + continueStructuredAgentSessionAfterRestart(deps, SESSION, marker(), 'operation-1') ).resolves.toEqual({ sessionId: SESSION, outcome: 'refused', @@ -98,7 +115,7 @@ it('reports an unattached chat without sending', async () => { deps.currentFence = () => null await expect( - continueStructuredAgentSessionAfterRestart(deps, SESSION, marker()) + continueStructuredAgentSessionAfterRestart(deps, SESSION, marker(), 'operation-1') ).resolves.toEqual({ sessionId: SESSION, outcome: 'refused', @@ -106,3 +123,45 @@ it('reports an unattached chat without sending', async () => { }) expect(deps.send).not.toHaveBeenCalled() }) + +// A start that failed rejects the continuation before any provider saw it: that is the verdict. +it('reports a continuation rejected at handover without waiting for the provider', async () => { + const deps = dependencies('accepted', 'rejected') + + await expect( + continueStructuredAgentSessionAfterRestart(deps, SESSION, marker(), 'operation-1') + ).resolves.toEqual({ sessionId: SESSION, outcome: 'refused', reason: 'Codex could not start.' }) + expect(deps.awaitSettlement).not.toHaveBeenCalled() + expect(deps.note).toHaveBeenCalledExactlyOnceWith(...REFUSED) +}) + +// The start completes once the agent took the message; the provider's answer is a later verdict. +it('returns at handover and leaves the provider verdict to be awaited', async () => { + const deps = dependencies('accepted') + + const started = await startStructuredAgentSessionContinuation( + deps, + SESSION, + marker(), + 'operation-1' + ) + + expect(deps.awaitHandedOver).toHaveBeenCalledOnce() + expect(deps.awaitSettlement).not.toHaveBeenCalled() + if (!('verdict' in started)) { + throw new Error('expected a handed-over continuation') + } + await expect(started.verdict()).resolves.toEqual({ sessionId: SESSION, outcome: 'continued' }) + expect(deps.note).toHaveBeenCalledOnce() +}) + +// A handover wait that ends without an answer — the session closed, or too many waited — proves +// nothing, so the provider's verdict still decides. +it('still awaits the verdict when the handover wait ends unanswered', async () => { + const deps = dependencies('unknown', undefined) + + await expect( + continueStructuredAgentSessionAfterRestart(deps, SESSION, marker(), 'operation-1') + ).resolves.toEqual({ sessionId: SESSION, outcome: 'unknown' }) + expect(deps.note).toHaveBeenCalledExactlyOnceWith(...UNCONFIRMED) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts index c025c0dc46af..e96f2b77c7f1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts @@ -1,9 +1,10 @@ // Asking an interrupted agent to carry on, on the user's opt-in. // -// Reattaching and continuing are SEPARATE operations: `resume` reattaches and sends nothing; this -// adds one message on top of it. Both the restart prompt and an opted-in launch come here, so a -// SETTING can reach this send — acceptable because the work is the user's own, the message asks the -// agent to verify its last action before repeating it, and the launch toast reports what happened. +// The continuation is a send like any other: accepted into the conversation, and delivered by the +// session's delivery loop, which starts the agent. Both the restart prompt and an opted-in launch +// come here, so a SETTING can reach this send — acceptable because the work is the user's own, the +// message asks the agent to verify its last action before repeating it, and the launch toast +// reports what happened. import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' import type { @@ -12,16 +13,14 @@ import type { AgentSessionSendResult } from '../../../shared/agent-session-wire' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import { AGENT_SESSION_RESTART_CONTINUATION_NOTE, AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE, AGENT_SESSION_RESTART_CONTINUATION_UNCONFIRMED_NOTE, - AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE, - restartContinuationMessage + AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE } from '../../../shared/agent-session-restart-continuation' import { AgentSessionPreDispatchError } from './structured-agent-session-operation-settlement' -import { createHash } from 'node:crypto' +import { restartContinuationEnvelope } from './structured-agent-session-restart-continuation-envelope' import type { AgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' /** @@ -44,7 +43,10 @@ export type StructuredAgentSessionContinuationOutcome = { /** The slice of the host one continuation needs. Structural so this module never imports the host. */ export type StructuredAgentSessionContinuationHost = { - sessions: ReadonlyMap + sessions: ReadonlyMap + /** The fence a conversation write carries; null when this host has no record of the session. + * The send opens the conversation itself, so a closed one still answers. */ + conversationFence: (sessionId: string) => number | null send: (input: { envelope: AgentSessionMutationEnvelope body: AgentJournalMessageItem @@ -54,15 +56,15 @@ export type StructuredAgentSessionContinuationHost = { sessionId: string, clientMessageId: string ) => Promise<{ value: AgentSessionSendResult } | undefined> + awaitSendHandedOver: ( + sessionId: string, + clientMessageId: string + ) => Promise<{ value: AgentSessionSendResult } | undefined> onNoteFailed: (sessionId: string, error: unknown) => void now: () => number - /** Whether the marker is still an offer, with the continuation's own submission set aside. - * Re-asked right before dispatch, so a newer user message refuses the send; a provider turn - * running then does not, since both providers queue a message sent mid-turn. */ - stillResumable: ( - marker: AgentSessionResumeMarker, - options: { pendingContinuationId: string } - ) => boolean + /** Whether the marker is still an offer. Asked at acceptance, inside the session lock, so the + * first message accepted since the restart decides: the user's, or this continuation. */ + stillResumable: (marker: AgentSessionResumeMarker) => boolean } /** Binds one continuation to the host: the superseded check before dispatch, the settlement @@ -72,52 +74,39 @@ export function restartContinuationDeps( marker: AgentSessionResumeMarker ): StructuredAgentSessionContinuationDeps { return { - currentFence: (sessionId) => host.sessions.get(sessionId)?.fence ?? null, + currentFence: host.conversationFence, send: (input) => host.send({ ...input, beforeRun: () => { - if ( - !host.stillResumable(marker, { - pendingContinuationId: input.envelope.clientOperationId - }) - ) { + if (!host.stillResumable(marker)) { throw new RestartContinuationSupersededError() } } }), awaitSettlement: async (sessionId, clientMessageId) => (await host.awaitSendSettlement(sessionId, clientMessageId))?.value.submission, + awaitHandedOver: async (sessionId, clientMessageId) => + (await host.awaitSendHandedOver(sessionId, clientMessageId))?.value.submission, onNoteFailed: host.onNoteFailed, note: restartNoteWriter(host) } } -/** The note for a reattach that failed before any continuation was attempted. */ -export function noteRestartReattachFailed( - host: StructuredAgentSessionContinuationHost, - sessionId: string -): Promise { - return noteNotContinued( - { note: restartNoteWriter(host), onNoteFailed: host.onNoteFailed }, - sessionId, - 'not-connected' - ) -} - /** Writes a host-authored status note into the chat. */ function restartNoteWriter( - host: Pick + host: Pick ): StructuredAgentSessionContinuationDeps['note'] { return async (sessionId, text, tone) => { const session = host.sessions.get(sessionId) - if (!session) { + const fence = host.conversationFence(sessionId) + if (!session || fence === null) { return } await session.journal.appendItem( { provider: 'orca', clientMessageId: `restart-continuation:${sessionId}:${host.now()}` }, { kind: 'status', text, ...(tone ? { tone } : {}) }, - { fence: session.fence } + { fence } ) } } @@ -129,59 +118,21 @@ const OWNERSHIP_REFUSALS = new Set([ 'execution_owner_reconciling' ]) +/** The user's own message was accepted first; the offer is spent, and nothing failed. */ +export const RESTART_CONTINUATION_SUPERSEDED = 'agent_session_restart_work_superseded' + /** Only this pre-dispatch failure proves a thrown send did not deliver. */ export class RestartContinuationSupersededError extends AgentSessionPreDispatchError { constructor() { - super('agent_session_restart_work_superseded') + super(RESTART_CONTINUATION_SUPERSEDED) this.name = 'RestartContinuationSupersededError' } } -/** The message body, built once so both the send and any test read the same text. */ -export function restartContinuationBody(marker: AgentSessionResumeMarker): AgentJournalMessageItem { - return { - kind: 'message', - role: 'user', - blocks: [{ type: 'text', text: restartContinuationMessage(marker) }] - } -} - -/** The fence only fills the envelope: admission names this send by its operation id, not a fence. */ -export function restartContinuationEnvelope( - sessionId: string, - fence: number, - marker: AgentSessionResumeMarker -): { envelope: AgentSessionMutationEnvelope; body: AgentJournalMessageItem } { - const body = restartContinuationBody(marker) - return { - body, - envelope: { - sessionId, - // The same interrupted work must reach the durable ledger with the same message identity. - clientOperationId: `${marker.recordedAt.toString().padStart(13, '0')}-${createHash('sha256') - .update( - JSON.stringify([ - marker.teardownId, - sessionId, - marker.work.kind, - marker.work.id, - marker.providerHandleRoot - ]) - ) - .digest('hex') - .slice(0, 32)}`, - expectedRuntimeFence: fence, - payloadFingerprint: computeAgentSessionPayloadFingerprint({ - method: 'agentSession.send', - sessionId, - fields: { body } - }) - } - } -} +type ContinuationSubmission = { dispatchState?: string; reason?: string | null } export type StructuredAgentSessionContinuationDeps = { - /** Runtime fence as it stands now; null when the session is not attached. */ + /** Runtime fence as it stands now; null when this host has no record of the session. */ currentFence: (sessionId: string) => number | null send: (input: { envelope: AgentSessionMutationEnvelope @@ -202,7 +153,13 @@ export type StructuredAgentSessionContinuationDeps = { awaitSettlement: ( sessionId: string, clientMessageId: string - ) => Promise<{ dispatchState?: string; reason?: string | null } | undefined> + ) => Promise + /** Waits until the send is handed to a started agent or rejected. Undefined when the wait ended + * first — the session closed, or too many waited — which proves neither. */ + awaitHandedOver: ( + sessionId: string, + clientMessageId: string + ) => Promise /** Records a host-authored journal note: that this send was Orca's, not the user's, or that the * chat did not carry on. `tone` is a display hint older clients render as plain text. */ note: (sessionId: string, text: string, tone?: 'error' | 'warning') => Promise @@ -211,26 +168,61 @@ export type StructuredAgentSessionContinuationDeps = { onNoteFailed: (sessionId: string, error: unknown) => void } +/** A continuation handed to its agent, or already decided. */ +export type StartedStructuredAgentSessionContinuation = + | { done: StructuredAgentSessionContinuationOutcome } + | { verdict: () => Promise } + /** - * Sends the continuation to ONE already-reconnected session. - * - * The caller must have reconnected it first: this deliberately does not reconnect, so that every - * eligibility check, the admission gate and the consume-once ordering stay in the resume path and - * are not re-implemented here. + * Sends the continuation to ONE session and returns once the agent has taken it or its start + * failed — the point a restart batch counts a start as done. What the provider then answered is + * the `verdict`, awaited separately so a slow answer does not hold the batch. */ -export async function continueStructuredAgentSessionAfterRestart( +export async function startStructuredAgentSessionContinuation( deps: StructuredAgentSessionContinuationDeps, sessionId: string, - marker: AgentSessionResumeMarker -): Promise { - let result: StructuredAgentSessionContinuationOutcome + marker: AgentSessionResumeMarker, + /** This action's continuation, as its offer recorded it. */ + continuationId: string +): Promise { + let started: StartedStructuredAgentSessionContinuation try { - result = await sendContinuation(deps, sessionId, marker) + started = await sendContinuation(deps, sessionId, marker, continuationId) } catch (error) { - await noteNotContinued(deps, sessionId, 'refused') + // The user's own message came first: nothing failed, so the chat says nothing. + if (!(error instanceof RestartContinuationSupersededError)) { + await noteNotContinued(deps, sessionId, 'refused') + } throw error } - if (result.outcome !== 'continued') { + if ('done' in started) { + await noteOutcome(deps, sessionId, started.done) + return started + } + const { verdict } = started + return { + verdict: async () => { + const outcome = await verdict() + await noteOutcome(deps, sessionId, outcome) + return outcome + } + } +} + +async function noteOutcome( + deps: Pick, + sessionId: string, + result: StructuredAgentSessionContinuationOutcome +): Promise { + if (result.outcome === 'continued') { + // Only an accepted dispatch gets the note: it is a durable claim that Orca asked this agent to + // carry on. Best effort — losing it must not turn a delivered continuation into a failure. + try { + await deps.note(sessionId, AGENT_SESSION_RESTART_CONTINUATION_NOTE) + } catch (error) { + deps.onNoteFailed(sessionId, error) + } + } else { await noteNotContinued( deps, sessionId, @@ -241,7 +233,6 @@ export async function continueStructuredAgentSessionAfterRestart( : 'refused' ) } - return result } /** The chat itself carries the failure, so it survives the toast, a dismissed record and a restart, @@ -269,13 +260,14 @@ async function noteNotContinued( async function sendContinuation( deps: StructuredAgentSessionContinuationDeps, sessionId: string, - marker: AgentSessionResumeMarker -): Promise { + marker: AgentSessionResumeMarker, + continuationId: string +): Promise { const fence = deps.currentFence(sessionId) if (fence === null) { - return { sessionId, outcome: 'refused', reason: 'agent_session_not_attached' } + return { done: { sessionId, outcome: 'refused', reason: 'agent_session_not_attached' } } } - const { envelope, body } = restartContinuationEnvelope(sessionId, fence, marker) + const { envelope, body } = restartContinuationEnvelope(sessionId, fence, marker, continuationId) const sent = await deps.send({ envelope, body }).catch((error: unknown) => { if (error instanceof AgentSessionPreDispatchError) { throw error @@ -285,46 +277,61 @@ async function sendContinuation( return null }) if (!sent) { - return { sessionId, outcome: 'unknown' } + return { done: { sessionId, outcome: 'unknown' } } } if (!sent.ok) { return { - sessionId, - outcome: 'refused', - reason: sent.refusal?.code ?? 'agent_session_send_failed' + done: { + sessionId, + outcome: 'refused', + reason: sent.refusal?.code ?? 'agent_session_send_failed' + } } } - // The send result carries the dispatch as it stood when Orca took the message, which for a normal - // successful send is `pending`. Judging it here would report every delivered continuation as - // pending and never write the note, so the settled value is what decides. - const submission = - (await deps.awaitSettlement(sessionId, envelope.clientOperationId).catch(() => undefined)) ?? - sent.value?.submission + const clientMessageId = envelope.clientOperationId + const handedOver = await deps.awaitHandedOver(sessionId, clientMessageId).catch(() => undefined) + if (handedOver?.dispatchState === 'rejected') { + return { done: refusedBy(sessionId, handedOver) } + } + return { + verdict: async () => + verdictOf( + sessionId, + // The send result carries the dispatch as it stood when Orca took the message, which for + // a normal successful send is `pending`, so the settled value is what decides. + (await deps.awaitSettlement(sessionId, clientMessageId).catch(() => undefined)) ?? + handedOver ?? + sent.value?.submission + ) + } +} + +function refusedBy( + sessionId: string, + submission: ContinuationSubmission +): StructuredAgentSessionContinuationOutcome { + return { + sessionId, + outcome: 'refused', + reason: submission.reason ?? 'agent_session_dispatch_rejected' + } +} + +function verdictOf( + sessionId: string, + submission: ContinuationSubmission | undefined +): StructuredAgentSessionContinuationOutcome { const dispatch = submission?.dispatchState - if (dispatch === 'rejected') { - return { - sessionId, - outcome: 'refused', - reason: submission?.reason ?? 'agent_session_dispatch_rejected' - } + if (submission && dispatch === 'rejected') { + return refusedBy(sessionId, submission) } if (dispatch === 'pending') { // Still pending after settlement gave up: handed off, never confirmed. return { sessionId, outcome: 'pending' } } - if (dispatch !== 'accepted') { - // `unknown`, or a peer that reported no state at all. Delivery is unverifiable, so this claims - // neither success nor failure — and writes no note saying the agent was asked to continue. - return { sessionId, outcome: 'unknown' } - } - // Only an accepted dispatch gets the note: it is a durable claim that Orca asked this agent to - // carry on, and it must not sit beside a message the provider refused or never confirmed. Best - // effort beyond that — losing the note must not turn a delivered continuation into a failure — - // but reported, never swallowed. - try { - await deps.note(sessionId, AGENT_SESSION_RESTART_CONTINUATION_NOTE) - } catch (error) { - deps.onNoteFailed(sessionId, error) - } - return { sessionId, outcome: 'continued' } + // `unknown`, or a peer that reported no state at all: delivery is unverifiable, so this claims + // neither success nor failure — and writes no note saying the agent was asked to continue. + return dispatch === 'accepted' + ? { sessionId, outcome: 'continued' } + : { sessionId, outcome: 'unknown' } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts index 5dcc3ca7690c..f1fe0a40b421 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts @@ -2,21 +2,18 @@ import { afterEach, expect, it, vi } from 'vitest' import { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' import { AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE, - AGENT_SESSION_RESTART_CONTINUATION_UNCONFIRMED_NOTE, - AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE + AGENT_SESSION_RESTART_CONTINUATION_UNCONFIRMED_NOTE } from '../../../shared/agent-session-restart-continuation' -import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection' import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner' +import { STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER } from './structured-agent-session-restart-resume-wiring' import { interruptedRestart, statusNotes } from './structured-agent-session-restart-interruption-test-harness' +import { CALLER, envelope } from './structured-agent-session-host-test-harness' import { HOST_TEST_NOW as NOW, HOST_TEST_SESSION as SESSION, - HOST_TEST_THREAD as THREAD, hostTestMessage } from './structured-agent-session-host-test-data' @@ -24,206 +21,71 @@ import { afterEach(() => vi.restoreAllMocks()) -function providerEvents(acquire: Awaited>['acquire']) { - const events = acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') - } - return events -} - -// The reported case: after the reattach the provider replays a queued message, which the chat -// journals as a newer user turn, so the continuation is refused just before dispatch. -it('files a continuation superseded by a replayed message, lists it and says so in the chat', async () => { - const { host, acquire, dispatch, root } = await interruptedRestart() - await host.restartResume.list() - await host.hold(SESSION, 'pane') - const events = providerEvents(acquire) - const append = AgentSessionJournal.prototype.appendSubmission - vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockImplementationOnce( - async function (this: AgentSessionJournal, input) { - const cursor = await append.call(this, input) - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'replayed-turn', ordinal: 1 }, - hostTestMessage('A queued notification the provider replayed') - ) - return cursor - } - ) - - const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - - expect(dispatch).not.toHaveBeenCalled() - expect(result.resumed).toMatchObject([ - { outcome: 'refused', reason: 'agent_session_restart_work_superseded' } - ]) - const failure = { - sessionId: SESSION, - outcome: 'refused', - reason: 'agent_session_restart_work_superseded' - } - expect(result.failed).toMatchObject([failure]) - expect(await host.restartResume.listFailures()).toMatchObject([failure]) - expect(await new AgentSessionRecoveryCapsule(root).listFailed(NOW)).toHaveLength(1) - expect(statusNotes(host)).toContainEqual({ - text: AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE, - tone: 'error' - }) - host.release(SESSION, 'pane') -}) - -// Nothing was attempted and nothing is owed: the user moved on between listing and acting. +// Nothing was owed once the user's own message came first: the offer is spent and nothing is filed. it('files nothing for a chat the user moved on in before its attempt, and spends the offer', async () => { - const { host, acquire, root } = await interruptedRestart() + const { host, root, dispatch } = await interruptedRestart() await host.restartResume.list() - await host.hold(SESSION, 'pane') - const events = providerEvents(acquire) const admit = StructuredAgentSessionResumeAdmission.prototype.run vi.spyOn(StructuredAgentSessionResumeAdmission.prototype, 'run').mockImplementationOnce( async function (this, ...args) { - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'newer-turn', ordinal: 1 }, - hostTestMessage('A newer task from another client') - ) - await host.flushStreamedEvents(SESSION) + const body = hostTestMessage('A newer task from another client') + await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) return admit.apply(this, args) } ) const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(result.resumed).toMatchObject([{ reason: 'agent_session_resume_not_eligible' }]) + expect(result.continued.filter((entry) => entry.outcome === 'continued')).toEqual([]) expect(result.failed).toEqual([]) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) const capsule = new AgentSessionRecoveryCapsule(root) expect(await capsule.listFailed(NOW)).toEqual([]) expect(await capsule.list(NOW)).toEqual([]) - expect(statusNotes(host)).toEqual([]) - host.release(SESSION, 'pane') + expect(await statusNotes(host)).toEqual([]) }) -it.each(['resume', 'continueAfterRestart'] as const)( - 'says so in the chat when the reattach itself fails (%s)', - async (action) => { - const { host, acquire } = await interruptedRestart() - await host.restartResume.list() - acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) +// The continuation is accepted and its agent then fails to start: the message is rejected with the +// cause, the failure is filed, and the chat says the agent did not carry on. +it('says so in the chat when the agent cannot start for the continuation', async () => { + const { host, acquire } = await interruptedRestart() + await host.restartResume.list() + acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) - await host.restartResume[action]([SESSION], 'modal') + await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(await host.restartResume.listFailures()).toMatchObject([ - { sessionId: SESSION, outcome: 'refused' } - ]) - // The fix depends on why it failed, which the dialog explains; "send a message" would not work. - expect(statusNotes(host)).toEqual([ - { text: AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE, tone: 'error' } - ]) - } -) + expect(await host.restartResume.listFailures()).toMatchObject([ + { sessionId: SESSION, outcome: 'refused', retryable: true } + ]) + expect(await statusNotes(host)).toContainEqual({ + text: AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE, + tone: 'error' + }) +}) -/** A continuation the provider accepted whose settlement could not be written: filed unconfirmed. */ -async function unconfirmedContinuation() { - const state = await interruptedRestart() - const { host, store } = state +// A send that throws after Orca may have taken it cannot be proven undelivered: filed unconfirmed, +// and it stays on record while the agent that may be carrying on keeps running. +it('keeps an unconfirmed failure while the agent the continuation started keeps running', async () => { + const { host, store } = await interruptedRestart() vi.spyOn(console, 'warn').mockImplementation(() => {}) await host.restartResume.list() - await host.hold(SESSION, 'pane') const settle = store.recordOperationOutcome.bind(store) vi.spyOn(store, 'recordOperationOutcome').mockImplementation(async (input) => { - if (input.outcome.status === 'succeeded') { + if ( + input.callerKey === STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER && + input.outcome.status === 'succeeded' + ) { throw new Error('operation outcome could not be persisted') } return settle(input) }) + const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') + expect(result.failed).toMatchObject([{ sessionId: SESSION, outcome: 'unconfirmed' }]) - expect(statusNotes(host)).toContainEqual({ + expect(await statusNotes(host)).toContainEqual({ text: AGENT_SESSION_RESTART_CONTINUATION_UNCONFIRMED_NOTE, tone: 'warning' }) - const continuation = host.journalSnapshot(SESSION).submissions.at(-1) - const providerItemId = continuation?.providerItemId - if (!continuation || !providerItemId) { - throw new Error('missing accepted continuation') - } - return { ...state, continuation, providerItemId, events: providerEvents(state.acquire) } -} - -// The warning asked the user to check the agent's reply; a turn opened by the continuation's own -// message is that reply starting, however the provider names the message. -it.each(['submission key', 'provider key'] as const)( - 'retires an unconfirmed failure once the continuation opens a turn (%s)', - async (naming) => { - const { host, root, continuation, providerItemId, events } = await unconfirmedContinuation() - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'continued-turn', ordinal: 1 }, - { - kind: 'turn', - turnId: 'continued-turn', - state: 'running', - userItemId: - naming === 'submission key' - ? agentJournalSubmissionKey(continuation.clientMessageId) - : providerItemId - }, - { lifecycle: true } - ) - await host.flushStreamedEvents(SESSION) - - expect(await host.restartResume.listFailures()).toEqual([]) - await vi.waitFor(async () => { - expect(await new AgentSessionRecoveryCapsule(root).listFailed(NOW)).toEqual([]) - }) - host.release(SESSION, 'pane') - } -) - -it('keeps an unconfirmed failure while the newest turn is not the continuation’s', async () => { - const { host, events } = await unconfirmedContinuation() - // Provider output opened this turn, keyed by its own row rather than by any user message. - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'provider-turn', ordinal: 1 }, - { kind: 'turn', turnId: 'provider-turn', state: 'running', userItemId: 'provider-turn-row' }, - { lifecycle: true } - ) - await host.flushStreamedEvents(SESSION) - - expect(await host.restartResume.listFailures()).toMatchObject([{ outcome: 'unconfirmed' }]) - host.release(SESSION, 'pane') -}) - -// Nothing journaled the continuation, so the newest user message is still the interrupted one and -// the turn it opened is the interrupted work reporting in, not the agent carrying on. -it('keeps an unconfirmed failure whose continuation was never journaled while the interrupted turn runs', async () => { - const { host, acquire, store, marker } = await interruptedRestart('submission', false) - vi.spyOn(console, 'warn').mockImplementation(() => {}) - await host.restartResume.list() - await host.hold(SESSION, 'pane') - const events = providerEvents(acquire) - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'original-turn', ordinal: 1 }, - hostTestMessage('Perform the original task') - ) - await host.flushStreamedEvents(SESSION) - // A send that throws before recording anything cannot be proven undelivered. - vi.spyOn(store, 'admitMutationOperation').mockRejectedValueOnce(new Error('store unavailable')) - const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(result.failed).toMatchObject([{ sessionId: SESSION, outcome: 'unconfirmed' }]) - const submissions = host.journalSnapshot(SESSION).submissions - const original = submissions[0]?.providerItemId - if (submissions.length !== 1 || !original) { - throw new Error('expected only the original submission, accepted by the provider') - } - expect(latestStructuredAgentSessionUserItem(host.journalSnapshot(SESSION).items)?.itemId).toBe( - marker?.latestUserItemId - ) - - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'original-turn', ordinal: 2 }, - { kind: 'turn', turnId: 'original-turn', state: 'running', userItemId: original }, - { lifecycle: true } - ) - await host.flushStreamedEvents(SESSION) - expect(await host.restartResume.listFailures()).toMatchObject([{ outcome: 'unconfirmed' }]) - host.release(SESSION, 'pane') }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-ledger.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-ledger.ts index 3b8fad40de09..9b5a89c313ae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-ledger.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-ledger.ts @@ -4,9 +4,8 @@ // the offer, so without this record nothing durable would point at the chat the user has to // continue by hand. The capsule holds the record; this decides what goes in and when it leaves. // -// Whether a record is still current is derived, never cached: it is current only while the chat's -// newest user message is the one it had when the failure was filed. The user's own send, from any -// client, therefore retires it without a hook on the send path. +// A record ends when the chat's agent is started again outside a resume action — the host retires +// it at that start, with the offer — or by a successful retry, or a dismissal. import type { AgentSessionRecoveryCapsule, @@ -18,19 +17,10 @@ import type { AgentSessionResumeFailureOutcome, AgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' -import type { AgentJournalSnapshot } from '../../../shared/agent-session-journal-types' -import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' -import { isRootAgentJournalItem } from '../../../shared/agent-session-journal-producer' -import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' -import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection' import { normalizeOptionalField } from '../../../shared/agent-status-field-normalization' import { AGENT_MODEL_MAX_LENGTH } from '../../../shared/agent-status-types' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { adapterSupportsRecord } from './structured-agent-session-provider-support' -import { - liveStructuredAgentSessionLatestUserItemId, - type StructuredAgentSessionRestartJournalSource -} from './structured-agent-session-restart-candidates' import type { StructuredAgentSessionContinuationOutcome } from './structured-agent-session-restart-continuation' import type { StructuredAgentSessionResumeCandidate, @@ -40,37 +30,18 @@ import { STRUCTURED_AGENT_SESSION_RESUME_NOT_ELIGIBLE, type StructuredAgentSessionResumeOutcome } from './structured-agent-session-restart-resume-runner' +import { RESTART_CONTINUATION_SUPERSEDED } from './structured-agent-session-restart-continuation' type FailureCapsule = Pick< AgentSessionRecoveryCapsule, - | 'listFailed' - | 'completeResume' - | 'failResume' - | 'rollbackResume' - | 'dismiss' - | 'clearAll' - | 'forgetSuperseded' + 'listFailed' | 'completeResume' | 'failResume' | 'rollbackResume' | 'dismiss' | 'clearAll' > -/** What a failure is filed against: the chat's newest user message as its own attempt ended. Kept - * per chat rather than read at settlement, because the rest of a batch can take a while and a - * message the user sends meanwhile answers the failure rather than belongs to it. */ -export type StructuredAgentSessionRestartAttempts = { - observe: (sessionId: string) => void - latestUserItemId: (sessionId: string) => string | null -} - export type StructuredAgentSessionRestartFailureLedger = { /** The stored records, current or not. */ read: () => Promise - /** The current records as rows a surface can show; sessions this host no longer holds are left - * out, and records the chat has since superseded are dropped and pruned. */ + /** The records as rows a surface can show; sessions this build cannot run are left out. */ list: () => Promise - /** One action's attempts; a chat never observed after an attempt falls back to its reserved - * marker. */ - attempts: ( - markers: ReadonlyMap - ) => StructuredAgentSessionRestartAttempts /** Settles one operation's reservations: the agent carried on, or the failure is filed. Rows the * operation still owns after that are reopened. */ settle: ( @@ -78,7 +49,8 @@ export type StructuredAgentSessionRestartFailureLedger = { outcomes: readonly StructuredAgentSessionResumeOutcome[], action: { candidates: readonly StructuredAgentSessionResumeCandidate[] - attempts: StructuredAgentSessionRestartAttempts + /** The reserved markers; a failure carries its marker's message id for older readers. */ + markers: ReadonlyMap /** How a reattached session's action ended; null when the agent carried on. Reattaching alone * is not the whole action, so the runner's own outcome cannot decide this. */ failureAfterResume: (sessionId: string) => AgentSessionResumeFailureOutcome | null @@ -99,54 +71,14 @@ export function continuationFailureOutcome( return outcome === 'continued' ? null : outcome === 'refused' ? 'refused' : 'unconfirmed' } -/** Whether the chat itself has moved past a failure: a newer user message, or, for a delivery - * nobody confirmed, a turn the continuation's own message opened. */ -function failureAnsweredByChat( - failure: AgentSessionResumeFailureRecord, - snapshot: AgentJournalSnapshot -): boolean { - const latest = latestStructuredAgentSessionUserItem(snapshot.items)?.itemId ?? null - if (latest !== failure.latestUserItemId) { - return true - } - // Only when the continuation's message was journaled: otherwise the newest user message is the - // interrupted one, whose own turn proves nothing about the continuation. - return ( - failure.outcome === 'unconfirmed' && - latest !== null && - latest !== failure.marker.latestUserItemId && - newestTurnUserItemId(snapshot) === latest - ) -} - -/** The user message that opened the newest root turn, resolved through a provider key the send was - * accepted under. */ -function newestTurnUserItemId(snapshot: AgentJournalSnapshot): string | null { - for (let index = snapshot.items.length - 1; index >= 0; index -= 1) { - const item = snapshot.items[index] - const turn = isRootAgentJournalItem(item) ? readAgentJournalTurn(item?.body) : null - if (!turn) { - continue - } - const key = turn.userItemId - if (key === undefined) { - return null - } - const accepted = snapshot.submissions.find((entry) => entry.providerItemId === key) - return accepted ? agentJournalSubmissionKey(accepted.clientMessageId) : key - } - return null -} - export function createStructuredAgentSessionRestartFailureLedger(deps: { capsule?: FailureCapsule - sessions: ReadonlyMap - /** Makes a persisted chat's journal readable here, as listing an offer does. */ - reveal: (sessionId: string) => Promise getRecord: (sessionId: string) => AgentSessionRecord | null adapter: StructuredAgentSessionAdapter /** The predicate a retry applies to the failure's marker. */ retryable: (marker: AgentSessionResumeMarker) => boolean + /** Makes the failed chats readable here, so `retryable` reads each one's journal. */ + reveal: (markers: readonly AgentSessionResumeMarker[]) => Promise now: () => number /** The capsule's single mutation lane, shared with the offer's own operations. */ enqueue: (operation: () => Promise) => Promise @@ -190,35 +122,9 @@ export function createStructuredAgentSessionRestartFailureLedger(deps: { } const list = async (): Promise => { - const current: AgentSessionResumeFailureRecord[] = [] - const superseded: AgentSessionResumeFailureRecord[] = [] - for (const failure of await read()) { - const sessionId = failure.marker.sessionId - if (!deps.sessions.has(sessionId)) { - await deps.reveal(sessionId) - } - const snapshot = deps.sessions.get(sessionId)?.journal.snapshot() - // An unreadable journal decides nothing; the record stays until something that can decide. - if (snapshot && failureAnsweredByChat(failure, snapshot)) { - superseded.push(failure) - } else { - current.push(failure) - } - } - const capsule = deps.capsule - if (capsule && superseded.length > 0) { - const gone = superseded.map((failure) => ({ - sessionId: failure.marker.sessionId, - recordedAt: failure.marker.recordedAt, - failedAt: failure.failedAt - })) - void deps - .enqueue(() => capsule.forgetSuperseded(gone, deps.now())) - .catch(() => { - console.warn('[structured-agent-session] pruning superseded restart failures failed') - }) - } - return current.flatMap(toRow) + const failures = await read() + await deps.reveal(failures.map((failure) => failure.marker)) + return failures.flatMap(toRow) } const settle: StructuredAgentSessionRestartFailureLedger['settle'] = async ( @@ -232,16 +138,22 @@ export function createStructuredAgentSessionRestartFailureLedger(deps: { } const completed: string[] = [] const failures: AgentSessionResumeFailureInput[] = [] - const promptBySession = new Map( - action.candidates.map((candidate) => [candidate.sessionId, candidate.latestPrompt]) - ) + // A retry keeps the prompt its first failure named: the chat's newest user message since then + // is the rejected continuation, and a message of the user's own would have ended the offer. + const promptBySession = new Map([ + ...action.candidates.map( + (candidate) => [candidate.sessionId, candidate.latestPrompt] as const + ), + ...(await read()).map((filed) => [filed.marker.sessionId, filed.latestPrompt] as const) + ]) for (const outcome of outcomes) { const resumed = outcome.outcome === 'resumed' - // Ineligible means the user moved on or the offer no longer applies (record gone, - // conversation forked), so there is nothing to retry and the offer is spent. + // Ineligible means the offer no longer applies (record gone, conversation forked), and + // superseded means the user's own message came first: nothing to retry, and the offer is spent. const failure = resumed ? action.failureAfterResume(outcome.sessionId) - : outcome.reason === STRUCTURED_AGENT_SESSION_RESUME_NOT_ELIGIBLE + : outcome.reason === STRUCTURED_AGENT_SESSION_RESUME_NOT_ELIGIBLE || + outcome.reason === RESTART_CONTINUATION_SUPERSEDED ? null : 'refused' if (failure === null) { @@ -256,7 +168,7 @@ export function createStructuredAgentSessionRestartFailureLedger(deps: { ? action.failureReason(outcome.sessionId) : (outcome.reason ?? 'agent_session_resume_refused'), latestPrompt: promptBySession.get(outcome.sessionId) ?? '', - latestUserItemId: action.attempts.latestUserItemId(outcome.sessionId) + latestUserItemId: action.markers.get(outcome.sessionId)?.latestUserItemId ?? null }) } await deps @@ -280,27 +192,9 @@ export function createStructuredAgentSessionRestartFailureLedger(deps: { }) } - const attempts: StructuredAgentSessionRestartFailureLedger['attempts'] = (markers) => { - const observed = new Map() - return { - // Observed after the attempt, so the continuation's own message is part of the filed state. - observe: (sessionId) => { - const latest = liveStructuredAgentSessionLatestUserItemId(deps.sessions, sessionId) - if (latest !== undefined) { - observed.set(sessionId, latest) - } - }, - latestUserItemId: (sessionId) => { - const latest = observed.get(sessionId) - return latest !== undefined ? latest : (markers.get(sessionId)?.latestUserItemId ?? null) - } - } - } - return { read, list, - attempts, settle, dismiss: (sessionIds, beforeClearAll) => deps.enqueue(async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts index 1bcb27ca4a2e..4b10484a24bf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts @@ -10,7 +10,6 @@ import { } from '../../runtime/agent-session-recovery-capsule' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { parseAgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner' import { @@ -25,10 +24,22 @@ import { HOST_TEST_NOW as NOW, HOST_TEST_SESSION as SESSION, HOST_TEST_THREAD as THREAD, + hostTestAttachParams, hostTestMessage } from './structured-agent-session-host-test-data' -export const GRACE = 15_000 +/** Starts the agent explicitly — the attach a client's ensure makes — for a test that needs a + * running child before its next step. Nothing else starts one ahead of a send. */ +export async function startAgent(state: { + host: StructuredAgentSessionHost + store: AgentSessionRecordStore +}): Promise { + const result = await state.host.attach( + CALLER, + hostTestAttachParams(state.store.getRecord(SESSION)?.lease.runtimeFence ?? null) + ) + expect(result.ok).toBe(true) +} export async function interruptedRestart( work: 'turn' | 'submission' | 'send-after-reply' | 'children' = 'turn', @@ -52,6 +63,8 @@ export async function interruptedRestart( previous.dispatch.mockResolvedValueOnce({ state: 'admitted' }) const body = hostTestMessage('Perform the original task') await previous.host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + // Accepted first, handed over after: the work in flight is a send the provider took. + await vi.waitFor(() => expect(previous.dispatch).toHaveBeenCalledOnce()) } else if (work === 'children') { events.appendItem( { provider: 'codex', threadId: THREAD, turnId: 'settled-turn', ordinal: 1 }, @@ -97,6 +110,8 @@ export async function interruptedRestart( hostId: 'local' }) const closeSession = vi.fn(async () => true) + // The relaunch comes after the quit that recorded the offer. + const clock = { now: NOW + 1 } const host = new StructuredAgentSessionHost({ store, adapter: { @@ -117,8 +132,7 @@ export async function interruptedRestart( mintSpawnToken: () => 'spawn-next', probeOwner: async () => ({ outcome: 'pid-absent' }), recoveryCapsule: new AgentSessionRecoveryCapsule(previous.root), - releaseGraceMs: GRACE, - now: () => NOW + now: () => clock.now }) replaceHostTestState({ store, host }) previous.acquire.mockClear() @@ -128,43 +142,34 @@ export async function interruptedRestart( await readFile(join(previous.root, AGENT_SESSION_RECOVERY_CAPSULE_FILE), 'utf8') ) const marker = parseAgentSessionResumeMarker(capsule.entries[0]?.marker) - return { ...hostTestState(), host, store, closeSession, marker } + return { ...hostTestState(), host, store, closeSession, marker, clock } } -export function statusNotes(host: StructuredAgentSessionHost) { - return host - .journalSnapshot(SESSION) - .items.flatMap((item) => - item.body.kind === 'status' ? [{ text: item.body.text, tone: item.body.tone }] : [] - ) +export async function statusNotes(host: StructuredAgentSessionHost) { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' ? [{ text: item.body.text, tone: item.body.tone }] : [] + ) } -/** A reattach that succeeds and a continuation the host refuses: a message from another client - * lands while the continuation is being recorded. `userAnswers` has the user reply in the chat - * just before or after its own attempt, while the rest of a batch would still be running. */ +/** A continuation the host refuses because the user's own message was accepted first: another + * client's send lands after the action reserved the offer, just before the continuation is + * accepted. `userAnswers` instead has the user send before or after the whole attempt. */ export async function supersededRefusal(userAnswers?: 'before' | 'after') { - const { host, acquire, dispatch, root } = await interruptedRestart() + const { host, store, acquire, dispatch, root } = await interruptedRestart() await host.restartResume.list() - await host.hold(SESSION, 'pane') - const events = acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') + const body = hostTestMessage('A newer task from another client') + const answer = () => + host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + const send = host.send + const writing = vi.spyOn(host, 'send') + if (!userAnswers) { + writing.mockImplementationOnce(async (caller, params) => { + await answer() + return send(caller, params) + }) } - const append = AgentSessionJournal.prototype.appendSubmission - const writing = vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission') - writing.mockImplementationOnce(async function (this: AgentSessionJournal, input) { - const cursor = await append.call(this, input) - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'newer-turn', ordinal: 1 }, - hostTestMessage('A newer task from another client') - ) - return cursor - }) const admit = StructuredAgentSessionResumeAdmission.prototype.run const admitting = vi.spyOn(StructuredAgentSessionResumeAdmission.prototype, 'run') - const body = hostTestMessage('Carry on from where you stopped') - const answer = () => - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) if (userAnswers) { admitting.mockImplementationOnce(async function (this, ...args) { await (userAnswers === 'before' ? answer() : null) @@ -177,9 +182,7 @@ export async function supersededRefusal(userAnswers?: 'before' | 'after') { } try { const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(result.continued).toMatchObject([{ outcome: 'refused' }]) - expect(dispatch).toHaveBeenCalledTimes(userAnswers ? 1 : 0) - return { host, root, result } + return { host, store, acquire, dispatch, root, result } } finally { writing.mockRestore() admitting.mockRestore() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-after-reattach.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-after-reattach.test.ts index 912594a1737b..032b0e07b2af 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-after-reattach.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-after-reattach.test.ts @@ -1,99 +1,42 @@ -// An offer across the provider reattaching and rewriting the chat in its own words: a notice turn -// of its own, restated rows. None of it withdraws the offer or changes the message Resume sends. +// An offer across a failed attempt and its retry, and across builds: the message Resume sends +// depends on the marker alone, and each action sends its own. import { expect, it } from 'vitest' import { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' import { AGENT_SESSION_RESTART_CONTINUATION_MESSAGE } from '../../../shared/agent-session-restart-continuation' -import { restartContinuationBody } from './structured-agent-session-restart-continuation' +import { restartContinuationBody } from './structured-agent-session-restart-continuation-envelope' import { interruptedRestart } from './structured-agent-session-restart-interruption-test-harness' import { HOST_TEST_NOW as NOW, - HOST_TEST_SESSION as SESSION, - HOST_TEST_THREAD as THREAD + HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' -function resumedEvents(state: Awaited>) { - // The latest acquisition: an earlier one may have been refused. - const events = state.acquire.mock.calls.at(-1)?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') - } - return events -} - -// THE REPORTED REFUSAL: opening the chat reattached Claude, which opened a turn of its own to say -// the previous session did not finish, and closed it. The chat is still offered and Resume sends — -// including when the offer is a send that had not become a turn, where that closed notice turn is -// the newest turn in the journal. -it.each(['turn', 'submission'] as const)( - 'still offers and continues a %s offer after the provider opens and closes a notice turn', - async (work) => { - const state = await interruptedRestart(work, false) - const { host, dispatch } = state - await host.hold(SESSION, 'pane') - const events = resumedEvents(state) - const notice = { provider: 'codex', threadId: THREAD, turnId: 'notice-turn' } as const - events.appendItem( - { ...notice, ordinal: 1 }, - { kind: 'turn', turnId: 'notice-turn', state: 'running', userItemId: 'turn:notice-turn' } - ) - events.appendItem( - { ...notice, ordinal: 2 }, - { - kind: 'message', - role: 'assistant', - blocks: [{ type: 'text', text: "didn't finish before the previous session ended" }] - } - ) - events.appendItem( - { ...notice, ordinal: 1 }, - { kind: 'turn', turnId: 'notice-turn', state: 'completed', userItemId: 'turn:notice-turn' } - ) - await host.flushStreamedEvents(SESSION) - - expect(await host.restartResume.list()).toMatchObject([{ sessionId: SESSION }]) - expect( - (await host.restartResume.continueAfterRestart([SESSION], 'modal')).continued - ).toMatchObject([{ outcome: 'continued' }]) - expect(dispatch).toHaveBeenCalledTimes(1) - host.release(SESSION, 'pane') - } -) - -// A retry after the first attempt never reached the provider, with the rows restated in between: -// the body depends on the marker alone, so the ledger fingerprint stays the offer's. -it('sends the same continuation body on a retry after the provider restates its rows', async () => { - const state = await interruptedRestart('children') - const { host, acquire, dispatch, marker } = state +// The first attempt's start failed, so its continuation was rejected and never reached the agent. +// A retry is a new action with a new message, not a replay of the rejected one, and it is +// delivered with the same body. +it("delivers a retry as a new continuation after the first one's start failed", async () => { + const { host, acquire, dispatch, marker } = await interruptedRestart('children') if (!marker) { throw new Error('missing interrupted restart marker') } acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) - await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(await host.restartResume.listFailures()).toMatchObject([{ retryable: true }]) + const first = await host.restartResume.continueAfterRestart([SESSION], 'modal') + expect(first.failed).toMatchObject([{ sessionId: SESSION, outcome: 'refused', retryable: true }]) + const [rejected] = (await host.journalSnapshot(SESSION)).submissions + expect(rejected).toMatchObject({ dispatchState: 'rejected' }) - await host.hold(SESSION, 'pane') - resumedEvents(state).appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'settled-turn', ordinal: 2 }, - { - kind: 'message', - role: 'system', - blocks: [ - { - type: 'subagent-group', - groupId: 'settled-turn', - agents: [{ id: 'child-1', label: 'Review loop 4', state: 'completed' }] - } - ] - } - ) - await host.flushStreamedEvents(SESSION) const retried = await host.restartResume.continueAfterRestart([SESSION], 'retry') expect(retried.continued).toMatchObject([{ outcome: 'continued' }]) - expect(dispatch).toHaveBeenCalledTimes(1) - expect(dispatch.mock.calls[0]?.[0].body).toEqual(restartContinuationBody(marker)) - host.release(SESSION, 'pane') + expect(dispatch).toHaveBeenCalledOnce() + const sent = dispatch.mock.calls[0]?.[0] + expect(sent?.clientMessageId).not.toBe(rejected?.clientMessageId) + expect(sent?.body).toEqual(restartContinuationBody(marker)) + expect((await host.journalSnapshot(SESSION)).submissions).toMatchObject([ + { clientMessageId: rejected?.clientMessageId, dispatchState: 'rejected' }, + { clientMessageId: sent?.clientMessageId, dispatchState: 'accepted' } + ]) + expect(retried.failed).toEqual([]) }) // A marker from a build that recorded only a working lead: no snapshot, so no activity to name, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-endings.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-endings.test.ts index 69f6f661c370..aefa34f4b673 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-endings.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-endings.test.ts @@ -1,10 +1,11 @@ -// The ways an offer ends. Every ending is the user's own act, and every ending DELETES the -// durable record — nothing stays behind to be re-filtered on every later read. +// The ways an offer ends. Every ending DELETES the durable record — nothing stays behind to be +// re-filtered on every later read. import { expect, it, vi } from 'vitest' import { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' -import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' +import { rename, writeFile, rm } from 'node:fs/promises' import { interruptedRestart } from './structured-agent-session-restart-interruption-test-harness' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import { CALLER, envelope } from './structured-agent-session-host-test-harness' import { HOST_TEST_NOW as NOW, @@ -12,36 +13,47 @@ import { hostTestMessage } from './structured-agent-session-host-test-data' -// The user answering the chat themselves is the one signal the offer is moot. The next listing -// notices and deletes the durable record, not merely hides it. +// The user's own message starts the chat's agent again, which ends the offer: the durable record +// is deleted, not merely hidden (R-04). it('deletes the offer once the user sends their own message in that chat', async () => { const { host, root, dispatch } = await interruptedRestart() const capsule = new AgentSessionRecoveryCapsule(root) expect(await capsule.list(NOW)).toHaveLength(1) - await host.hold(SESSION, 'pane') dispatch.mockResolvedValueOnce({ state: 'admitted' }) const body = hostTestMessage('Never mind, do this instead') await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + // Written when the agent's start is proven: a cold start. + await vi.waitFor( + async () => { + expect(await capsule.list(NOW)).toEqual([]) + }, + { timeout: 10_000 } + ) expect(await host.restartResume.list()).toEqual([]) - await vi.waitFor(async () => { - expect(await capsule.list(NOW)).toEqual([]) - }) - host.release(SESSION, 'pane') }) -// A journal this host cannot read says nothing about the user moving on, so it must not end the offer. +// A journal this host cannot read decides nothing, so it must not end the offer. it('keeps the offer when the chat cannot be read on this host', async () => { - const { host, root } = await interruptedRestart('submission') + const { host, root, store } = await interruptedRestart('submission') const capsule = new AgentSessionRecoveryCapsule(root) - const restoring = vi - .spyOn(StructuredAgentSessionReadableRestorer.prototype, 'restoreOne') - .mockRejectedValue(new Error('journal unreadable')) + const location = store.getRecord(SESSION)?.location + if (!location) { + throw new Error('missing session record') + } + // A file where the journal directory belongs: this host cannot open the conversation at all. + const journalDir = journalDirectoryFor(root, { + workspaceId: location.workspaceId, + sessionId: SESSION + }) + await rename(journalDir, `${journalDir}.aside`) + await writeFile(journalDir, 'not a journal') try { expect(await host.restartResume.list()).toMatchObject([{ sessionId: SESSION }]) } finally { - restoring.mockRestore() + await rm(journalDir) + await rename(`${journalDir}.aside`, journalDir) } expect(await capsule.list(NOW)).toHaveLength(1) expect(await host.restartResume.list()).toMatchObject([{ sessionId: SESSION }]) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-withdrawal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-withdrawal.test.ts new file mode 100644 index 000000000000..a0c0d80b2e13 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-withdrawal.test.ts @@ -0,0 +1,508 @@ +// A restart offer ends when the chat moves on after the restart: another message accepted, or its +// agent started, other than by the offer's own continuation. Each case reads the offer list and the +// capsule, never only an in-memory set. + +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + AgentSessionRecoveryCapsule, + AGENT_SESSION_RECOVERY_CAPSULE_FILE +} from '../../runtime/agent-session-recovery-capsule' +import { parseAgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { + interruptedRestart, + statusNotes +} from './structured-agent-session-restart-interruption-test-harness' +import { CALLER, envelope } from './structured-agent-session-host-test-harness' +import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idle-sweep' +import { createStructuredAgentSessionRestartOfferWithdrawal } from './structured-agent-session-restart-offer-withdrawal' +import { restartContinuationId } from './structured-agent-session-restart-continuation-envelope' +import { sweepOnce } from './structured-agent-session-rest-test-rig' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestMessage +} from './structured-agent-session-host-test-data' + +afterEach(() => vi.restoreAllMocks()) + +const SUPERSEDED = 'agent_session_restart_work_superseded' + +/** A start under a loaded machine: reconcile, acquire, hand over. */ +const COLD_START = { timeout: 10_000 } + +async function offered(work: 'turn' | 'submission' = 'turn') { + const state = await interruptedRestart(work, false) + expect(await state.host.restartResume.list()).toHaveLength(1) + return state +} + +function offersIn(root: string) { + return new AgentSessionRecoveryCapsule(root).list(NOW) +} + +function userSend(text: string) { + const body = hostTestMessage(text) + return { envelope: envelope('agentSession.send', { body }), body } +} + +function compactParams() { + return { + command: 'compact' as const, + envelope: envelope('agentSession.conversationCommand', { command: 'compact' }) + } +} + +function sentTexts(dispatch: Awaited>['dispatch']): string[] { + return dispatch.mock.calls.map(([input]) => + input.body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])).join('') + ) +} + +it('withdraws the offer when /compact starts the agent at rest (R-01)', async () => { + const { host, root, acquire } = await offered() + Object.assign(host.deps.adapter, { compact: vi.fn(async () => ({})) }) + + expect(await host.conversationCommand(CALLER, compactParams())).toMatchObject({ ok: true }) + + expect(acquire).toHaveBeenCalledOnce() + await vi.waitFor(async () => expect(await offersIn(root)).toEqual([]), COLD_START) + expect(await host.restartResume.list()).toEqual([]) +}) + +it('keeps the offer through every way of looking at the chat (R-02)', async () => { + const { host, root, acquire } = await offered() + const unsubscribe = await host.subscribe({ id: 'pane', sessionId: SESSION, emit: vi.fn() }) + await host.history({ sessionId: SESSION, direction: 'tail' }) + await host.readOptions(SESSION) + host.readCommands(SESSION) + await host.handoffStatus(SESSION) + await host.revealSession(SESSION) + unsubscribe() + + expect(acquire).not.toHaveBeenCalled() + expect(await host.restartResume.list()).toHaveLength(1) + expect(await offersIn(root)).toHaveLength(1) +}) + +it("does not let the offer's own continuation withdraw it (R-03)", async () => { + const { host, root, dispatch } = await offered() + const dismiss = vi.spyOn(AgentSessionRecoveryCapsule.prototype, 'dismiss') + + const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') + + expect(result.continued).toMatchObject([{ outcome: 'continued' }]) + expect(dispatch).toHaveBeenCalledOnce() + // The start was the continuation's own: whatever asked, it withdrew nothing. + for (const call of dismiss.mock.results) { + expect(await call.value).toBe(0) + } + // Ended by the success path instead. + expect(await offersIn(root)).toEqual([]) +}) + +it('refuses the continuation when another start lands inside its action (R-03b)', async () => { + const { host, root, dispatch } = await offered() + Object.assign(host.deps.adapter, { compact: vi.fn(async () => ({})) }) + const send = host.send + vi.spyOn(host, 'send').mockImplementationOnce(async (caller, params) => { + // Another client's /compact starts the agent after the action reserved the offer. + expect(await host.conversationCommand(CALLER, compactParams())).toMatchObject({ ok: true }) + return send(caller, params) + }) + + const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') + + expect(result.continued).toMatchObject([{ outcome: 'refused', reason: SUPERSEDED }]) + expect(dispatch).not.toHaveBeenCalled() + expect(await offersIn(root)).toEqual([]) + expect(result.failed).toEqual([]) +}) + +it('keeps the offer when a tabbed chat is restored at boot (R-05)', async () => { + const { host, store, root, acquire } = await offered() + await store.setSessionTabVisibility(SESSION, true) + + await host.restoreStartupSessions() + + expect(acquire).not.toHaveBeenCalled() + expect(await host.restartResume.list()).toHaveLength(1) + expect(await offersIn(root)).toHaveLength(1) +}) + +it('reads an older build’s marker whose message id no longer matches, and one with none (R-06)', async () => { + const { host, root, dispatch, marker } = await offered('submission') + if (!marker) { + throw new Error('missing interrupted restart marker') + } + // The older build compared this id with the chat's newest message; nothing does now. + await host.restartResume.dismiss([SESSION]) + await new AgentSessionRecoveryCapsule(root).record( + [{ ...marker, latestUserItemId: 'user-sent-under-the-older-build' }], + NOW + ) + expect(await host.restartResume.list()).toHaveLength(1) + const { latestUserItemId: _dropped, ...withoutId } = marker + expect(parseAgentSessionResumeMarker(withoutId)).toEqual(withoutId) + + await host.send(CALLER, userSend('Carry on')) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce(), COLD_START) + await vi.waitFor(async () => expect(await offersIn(root)).toEqual([]), COLD_START) +}) + +it('still writes the message id the previous build requires on every marker', async () => { + const { root } = await offered('submission') + const capsule = JSON.parse( + await readFile(join(root, AGENT_SESSION_RECOVERY_CAPSULE_FILE), 'utf8') + ) + expect(capsule.entries[0]?.marker).toHaveProperty('latestUserItemId') +}) + +/** Sends a message whose start fails, and waits for it to be rejected. */ +async function sendWhoseStartFails(state: Awaited>) { + state.acquire.mockRejectedValueOnce(new Error('Not signed in')) + const params = userSend('while signed out') + await state.host.send(CALLER, params) + await vi.waitFor(async () => + expect( + (await state.host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === params.envelope.clientOperationId + ) + ).toMatchObject({ dispatchState: 'rejected' }) + ) +} + +// A message the user sent is activity even when its start then failed. +it('withdraws the offer when the user sends a message whose start then fails (R-08)', async () => { + const state = await offered() + const { host, root, dispatch } = state + + await sendWhoseStartFails(state) + + expect(await host.restartResume.list()).toEqual([]) + await vi.waitFor(async () => expect(await offersIn(root)).toEqual([]), COLD_START) + // A click from a surface that still shows the offer finds nothing to act on. + expect(await host.restartResume.continueAfterRestart([SESSION], 'stale-click')).toMatchObject({ + resumed: [], + continued: [] + }) + expect(dispatch).not.toHaveBeenCalled() +}) + +// The idle sweep closes a chat's open handle, and the next read opens a new one: the user's message +// still counts, because it is read against where the journal stood when the offer was taken. +it('keeps the offer withdrawn after the idle sweep closes the chat and it is read again', async () => { + const state = await offered() + const { host, root, clock } = state + await sendWhoseStartFails(state) + clock.now += STRUCTURED_AGENT_SESSION_IDLE_MS + 1 + + await sweepOnce(host) + expect(host.hasSession(SESSION)).toBe(false) + await host.revealSession(SESSION) + + expect(await host.restartResume.list()).toEqual([]) + // The listing retires what it finds moved on; the write trails the answer. + await vi.waitFor(async () => expect(await offersIn(root)).toEqual([])) +}) + +// A failure the chat moved on from is read from its journal like an offer is, open or not. +it('keeps a failure not retryable after the user moved on and the sweep closed the chat', async () => { + const state = await offered() + const { host, clock } = state + state.acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) + const resumed = await host.restartResume.continueAfterRestart([SESSION], 'modal') + expect(resumed.failed).toMatchObject([{ sessionId: SESSION, retryable: true }]) + await sendWhoseStartFails(state) + expect(await host.restartResume.listFailures()).toMatchObject([{ retryable: false }]) + clock.now += STRUCTURED_AGENT_SESSION_IDLE_MS + 1 + + await sweepOnce(host) + expect(host.hasSession(SESSION)).toBe(false) + + expect(await host.restartResume.listFailures()).toMatchObject([{ retryable: false }]) +}) + +const DAY_AND_AN_HOUR = 25 * 60 * 60 * 1000 + +// An offer has no expiry; the ledger refuses a new id dated more than a day back. +it('resumes more than a day after the quit', async () => { + const state = await offered() + const { host, clock, dispatch } = state + clock.now += DAY_AND_AN_HOUR + + const resumed = await host.restartResume.continueAfterRestart([SESSION], 'modal') + + expect(resumed.continued).toMatchObject([{ sessionId: SESSION, outcome: 'continued' }]) + expect(sentTexts(dispatch)).toHaveLength(1) + expect(await host.restartResume.listFailures()).toEqual([]) +}) + +// The offer remembers its own continuations; nothing about them expires before the offer does. +it('keeps a failed resume retryable more than a day later, after the ledger pruned its row', async () => { + const state = await offered() + const { host, clock, store } = state + state.acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) + const first = await host.restartResume.continueAfterRestart([SESSION], 'modal') + expect(first.failed).toMatchObject([{ sessionId: SESSION, retryable: true }]) + clock.now += DAY_AND_AN_HOUR + // What an app restart does to the operation ledger a day on. + await store.reconcileOnRestart({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: reconciliation only reads the probe outcome. + probe: async () => ({ outcome: 'pid-absent' }) as never, + now: clock.now + }) + + expect(await host.restartResume.listFailures()).toMatchObject([ + { sessionId: SESSION, retryable: true } + ]) + const retried = await host.restartResume.continueAfterRestart([SESSION], 'retry') + expect(retried.continued).toMatchObject([{ sessionId: SESSION, outcome: 'continued' }]) +}) + +// The start was the continuation's own even after the provider refused it: the child it started, +// running or since stopped, is still the offer's, so the offer stays retryable. +it('keeps a resume retryable when its agent started but refused the continuation', async () => { + const state = await offered() + const { host, store } = state + const acquire = state.acquire.getMockImplementation() + if (!acquire) { + throw new Error('the harness acquire has no implementation') + } + state.acquire.mockImplementationOnce(async (input) => ({ + ...(await acquire(input)), + acquisitionGeneration: 'generation-continuation' + })) + state.dispatch.mockResolvedValueOnce({ + state: 'rejected', + reason: 'the provider refused the turn' + }) + + const first = await host.restartResume.continueAfterRestart([SESSION], 'modal') + + expect(state.acquire).toHaveBeenCalledOnce() + expect(first.failed).toMatchObject([{ sessionId: SESSION, retryable: true }]) + expect(await host.restartResume.listFailures()).toMatchObject([ + { sessionId: SESSION, retryable: true } + ]) + // The child the continuation started exits after proving its start. + await host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + fence: store.getRecord(SESSION)?.lease.runtimeFence ?? 0, + acquisitionGeneration: 'generation-continuation', + reason: 'codex app-server exited', + cause: 'unexpected-exit' + }) + expect(await host.restartResume.listFailures()).toMatchObject([ + { sessionId: SESSION, retryable: true } + ]) + const retried = await host.restartResume.continueAfterRestart([SESSION], 'retry') + expect(retried.continued).toMatchObject([{ sessionId: SESSION, outcome: 'continued' }]) +}) + +// The rejected continuation is the chat's newest user message; the row still names the user's. +it("names the user's prompt on a failed retry, not the rejected continuation", async () => { + const state = await offered('submission') + const { host } = state + state.acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) + const first = await host.restartResume.continueAfterRestart([SESSION], 'modal') + expect(first.failed).toMatchObject([{ latestPrompt: 'Perform the original task' }]) + + state.acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) + const retried = await host.restartResume.continueAfterRestart([SESSION], 'retry') + + expect(retried.failed).toMatchObject([ + { latestPrompt: 'Perform the original task', retryable: true } + ]) +}) + +// Resume that runs by itself at launch goes through the same call a click does, and the same rule +// decides: whichever of the two was accepted first since the restart wins. +it("refuses an automatic continuation when the user's message was accepted first, and writes nothing", async () => { + const { host, root, dispatch } = await offered() + const events: AgentSessionSubscribeEvent[] = [] + const unsubscribe = await host.subscribe({ + id: 'pane', + sessionId: SESSION, + emit: (event) => events.push(structuredClone(event)) + }) + const send = host.send + vi.spyOn(host, 'send').mockImplementationOnce(async (caller, params) => { + // Both accepted before any start, the user's first: the continuation queues behind the user's + // acceptance, ahead of the start that acceptance wakes. + let continuation: ReturnType | undefined + const append = AgentSessionJournal.prototype.appendSubmission + vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockImplementationOnce( + async function (this: AgentSessionJournal, ...args) { + continuation = send(caller, params) + await new Promise((resolve) => setTimeout(resolve, 50)) + return append.apply(this, args) + } + ) + await send(CALLER, userSend('A new request')) + return continuation! + }) + + const result = await host.restartResume.continueAfterRestart(undefined, 'launch') + + expect(result.continued).toMatchObject([{ outcome: 'refused', reason: SUPERSEDED }]) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce(), COLD_START) + expect(sentTexts(dispatch)).toEqual(['A new request']) + // Nothing the user did failed: no row reached the reader, and nothing is kept. + expect(await statusNotes(host)).toEqual([]) + expect( + events.some( + (event) => + event.type === 'batch' && event.batch.items.some((item) => item.body.kind === 'status') + ) + ).toBe(false) + expect(await offersIn(root)).toEqual([]) + expect(await new AgentSessionRecoveryCapsule(root).listFailed(NOW)).toEqual([]) + unsubscribe() +}) + +it('delivers a clicked continuation and a message sent right after it, in the order accepted', async () => { + const { host, root, dispatch } = await offered() + const send = host.send + vi.spyOn(host, 'send').mockImplementationOnce(async (caller, params) => { + const continuation = send(caller, params) + const user = send(CALLER, userSend('And then this')) + await user + return continuation + }) + + const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') + + expect(result.continued).toMatchObject([{ outcome: 'continued' }]) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(2), COLD_START) + const [first, second] = sentTexts(dispatch) + expect(first).not.toBe('And then this') + expect(second).toBe('And then this') + expect(await offersIn(root)).toEqual([]) +}) + +describe('reading the chat against where the offer was taken', () => { + const TAKEN = { epoch: 'epoch-1', sequence: 5 } + + const OFFER = { + sessionId: SESSION, + work: { kind: 'turn' as const, id: 'turn-1' }, + recordedAt: NOW, + trigger: 'quit' as const, + providerHandleRoot: 'codex:"thread"', + teardownId: 'teardown-1' + } + + function movedOn( + input: { + epoch?: string + submissions?: { clientMessageId: string; acceptedSequence: number; dispatchState: string }[] + }, + journalCursor: { epoch: string; sequence: number } | null = TAKEN + ): boolean { + const session = { + child: null, + journal: { + cursor: () => ({ epoch: input.epoch ?? TAKEN.epoch, sequence: 9 }), + submissions: () => input.submissions ?? [] + } + } + const withdrawal = createStructuredAgentSessionRestartOfferWithdrawal({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fact reads only the journal's cursor and submissions and the child fields given here. + sessions: new Map([[SESSION, session as never]]), + now: () => NOW, + enqueue: (operation) => operation() + }) + return withdrawal.movedOn({ ...OFFER, ...(journalCursor ? { journalCursor } : {}) }) + } + + it('counts a message accepted after that position, and nothing before it', () => { + const at = (acceptedSequence: number) => ({ + clientMessageId: `m-${acceptedSequence}`, + acceptedSequence, + dispatchState: 'rejected' + }) + expect(movedOn({ submissions: [at(4), at(5)] })).toBe(false) + expect(movedOn({ submissions: [at(4), at(6)] })).toBe(true) + }) + + it('counts a journal on another epoch as moved on', () => { + expect(movedOn({ epoch: 'epoch-2' })).toBe(true) + }) + + it("does not count the offer's own continuation while queued or rejected, so a retry still runs", () => { + const own = restartContinuationId(OFFER, 'operation-1', NOW) + const rejected = { clientMessageId: own, acceptedSequence: 7, dispatchState: 'rejected' } + expect(movedOn({ submissions: [rejected] })).toBe(false) + expect(movedOn({ submissions: [{ ...rejected, dispatchState: 'pending' }] })).toBe(false) + expect(movedOn({ submissions: [{ ...rejected, dispatchState: 'accepted' }] })).toBe(true) + // Another offer's continuation, and any other message, is the chat moving on. + const other = restartContinuationId({ ...OFFER, teardownId: 'teardown-2' }, 'operation-1', NOW) + expect(movedOn({ submissions: [{ ...rejected, clientMessageId: other }] })).toBe(true) + }) + + // The retry's reservation lapses with the app, and the capsule hands back the failure's marker, + // written before the retry ran. The next launch rejects the retry's queued continuation. + it('keeps a failure retryable after the app crashed during its retry', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-crash-during-retry-')) + try { + const capsule = new AgentSessionRecoveryCapsule(root) + await capsule.record([{ ...OFFER, journalCursor: TAKEN }], NOW) + await capsule.beginResume([SESSION], 'operation-a', NOW) + await capsule.failResume( + 'operation-a', + [ + { + sessionId: SESSION, + failedAt: NOW, + outcome: 'refused', + reason: 'x', + latestPrompt: '', + latestUserItemId: null + } + ], + NOW + ) + const [retrying] = await capsule.beginResume([SESSION], 'operation-b', NOW + 1) + if (!retrying) { + throw new Error('the retry reserved nothing') + } + const retry = restartContinuationId(retrying, 'operation-b', NOW + 1) + const afterCrash = NOW + 60 * 60_000 + expect(await capsule.list(afterCrash)).toEqual([]) + const [failed] = await capsule.listFailed(afterCrash) + + const session = { + child: null, + journal: { + cursor: () => ({ epoch: TAKEN.epoch, sequence: 9 }), + submissions: () => [ + { clientMessageId: retry, acceptedSequence: 7, dispatchState: 'rejected' } + ] + } + } + const withdrawal = createStructuredAgentSessionRestartOfferWithdrawal({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fact reads only the journal's cursor and submissions and the child fields given here. + sessions: new Map([[SESSION, session as never]]), + now: () => afterCrash, + enqueue: (operation) => operation() + }) + if (!failed) { + throw new Error('the failure did not outlive the crash') + } + expect(withdrawal.movedOn(failed.marker)).toBe(false) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('leaves an older build’s offer, which recorded no position, to a start', () => { + const after = { clientMessageId: 'm-7', acceptedSequence: 7, dispatchState: 'accepted' } + expect(movedOn({ submissions: [after], epoch: 'epoch-2' }, null)).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-withdrawal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-withdrawal.ts new file mode 100644 index 000000000000..020fb4bde511 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-withdrawal.ts @@ -0,0 +1,90 @@ +// When a restart offer ends without being acted on: the chat moved on. +// +// One derived fact decides it, for the offer list and for the continuation's own acceptance alike: +// since the offer was taken, another message was accepted in the chat, or its agent proved a start. +// A message is read against the journal position the offer recorded, so the answer survives any +// number of handle closes. A start is known only to the host that saw it, so it is also written +// to the recovery file when it happens. A restored row, a replayed row or a view carries neither, +// so opening a chat withdraws nothing. The rest of a resume action does not count: its own +// continuation, and the start that continuation waits on. + +import type { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' +import type { AgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { isRestartContinuationOf } from './structured-agent-session-restart-continuation-envelope' + +export type StructuredAgentSessionRestartOfferWithdrawal = ReturnType< + typeof createStructuredAgentSessionRestartOfferWithdrawal +> + +export type StructuredAgentSessionRestartOfferSession = Pick< + StructuredAgentSessionHostSession, + 'journal' | 'child' | 'lastEndedChild' +> + +export function createStructuredAgentSessionRestartOfferWithdrawal(deps: { + sessions: ReadonlyMap + capsule?: Pick + now: () => number + /** The capsule's single mutation lane, shared with the offer's own operations. */ + enqueue: (operation: () => Promise) => Promise +}) { + const movedOn = (marker: AgentSessionResumeMarker): boolean => { + const session = deps.sessions.get(marker.sessionId) + if (!session) { + return false + } + const taken = marker.journalCursor + // An older build's offer recorded no position: only a start withdraws it. + const accepted = + taken !== undefined && + (session.journal.cursor().epoch !== taken.epoch || + session.journal.submissions().some( + (submission) => + (submission.acceptedSequence ?? 0) > taken.sequence && + // The offer's own continuation, still queued or rejected, never reached the agent: a + // retry sends a new one. + !( + (submission.dispatchState === 'pending' || submission.dispatchState === 'rejected') && + isRestartContinuationOf(marker, submission.clientMessageId) + ) + )) + // Proven, not merely spawned: a start that failed during startup never ran the agent. Whose + // start it was is fixed when it was made, so a continuation rejected since still owns it. + const started = + session.child?.phase === 'ready' + ? session.child + : session.lastEndedChild?.duringStartup === false + ? session.lastEndedChild + : undefined + return ( + accepted || + (started !== undefined && + !(started.startedFor !== undefined && isRestartContinuationOf(marker, started.startedFor))) + ) + } + + return { + movedOn, + /** The chat's agent proved a start: the offer and any failure record go from the recovery file, + * unless the start was for one of that offer's own continuations. Advisory: a failed write is + * logged, never raised. */ + onAgentStarted: (sessionId: string): void => { + const session = deps.sessions.get(sessionId) + const capsule = deps.capsule + if (!capsule || !session) { + return + } + const startedFor = session.child?.startedFor + void deps + .enqueue(() => + capsule.dismiss([sessionId], deps.now(), (marker) => + startedFor === undefined ? false : isRestartContinuationOf(marker, startedFor) + ) + ) + .catch(() => { + console.warn('[structured-agent-session] withdrawing a restart offer failed') + }) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts index c3301861c010..a66eaa074320 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts @@ -7,8 +7,6 @@ import { parseAgentSessionResumeMarker } from '../../../shared/agent-session-res import { join } from 'node:path' import { afterEach, expect, it, vi } from 'vitest' import { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { pendingApproval } from './structured-agent-session-restart-resume-test-harness' -import { restartContinuationEnvelope } from './structured-agent-session-restart-continuation' import { AGENT_SESSION_RESTART_CONTINUATION_NOTE, AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE, @@ -16,8 +14,8 @@ import { } from '../../../shared/agent-session-restart-continuation' import { STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER } from './structured-agent-session-restart-resume-wiring' import { - GRACE, interruptedRestart, + startAgent, statusNotes, supersededRefusal } from './structured-agent-session-restart-interruption-test-harness' @@ -40,7 +38,7 @@ it('publishes continuation attribution to the subscribed chat without another pr const { host } = await interruptedRestart() await host.restartResume.list() const emit = vi.fn() - const unsubscribe = host.subscribe({ id: 'pane', sessionId: SESSION, emit }) + const unsubscribe = await host.subscribe({ id: 'pane', sessionId: SESSION, emit }) try { expect( (await host.restartResume.continueAfterRestart([SESSION], 'modal')).continued @@ -81,273 +79,75 @@ it.each(['turn', 'submission'] as const)( async (work) => { const { host, dispatch } = await interruptedRestart(work, false) expect(await host.restartResume.list()).toHaveLength(1) - await host.hold(SESSION, 'pane') + await startAgent(hostTestState()) dispatch.mockResolvedValueOnce({ state: 'admitted' }) const body = hostTestMessage('Stop the old task and do this instead') await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) await host.restartResume.continueAfterRestart([SESSION], 'modal') expect(dispatch).toHaveBeenCalledTimes(1) - expect(host.journalSnapshot(SESSION).submissions).toHaveLength(work === 'turn' ? 1 : 2) - host.release(SESSION, 'pane') - expect(host.isHeld(SESSION)).toBe(false) + expect((await host.journalSnapshot(SESSION)).submissions).toHaveLength(work === 'turn' ? 1 : 2) } ) -it('preserves the offer when the original submission receives its provider echo', async () => { - const { host, acquire, dispatch } = await interruptedRestart('submission', false) - expect(await host.restartResume.list()).toHaveLength(1) - await host.hold(SESSION, 'pane') - const events = acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') - } - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'original-turn', ordinal: 1 }, - hostTestMessage('Perform the original task') - ) - await host.flushStreamedEvents(SESSION) - expect(host.journalSnapshot(SESSION).submissions[0]?.dispatchState).toBe('accepted') - expect( - (await host.restartResume.continueAfterRestart([SESSION], 'modal')).continued - ).toMatchObject([{ outcome: 'continued' }]) - expect(acquire).toHaveBeenCalledTimes(1) - expect(dispatch).toHaveBeenCalledTimes(1) - host.release(SESSION, 'pane') - expect(host.isHeld(SESSION)).toBe(false) -}) - -// Teardown judged the chat working, and only the user moving on withdraws that. A send the -// provider proves it never received is still the chat Orca stopped; the continuation asks the agent +// Teardown judged the chat working. A send the provider proves it never received is still the chat +// Orca stopped; the continuation asks the agent // to check what finished rather than refusing. it('still continues a chat whose last send acquisition proves was never delivered', async () => { const { host, acquire, dispatch } = await interruptedRestart('submission') expect(await host.restartResume.list()).toHaveLength(1) const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(host.journalSnapshot(SESSION).submissions[0]).toMatchObject({ + expect((await host.journalSnapshot(SESSION)).submissions[0]).toMatchObject({ dispatchState: 'rejected', reason: 'not_delivered' }) expect(acquire).toHaveBeenCalledTimes(1) expect(dispatch).toHaveBeenCalledTimes(1) expect(result.continued).toMatchObject([{ outcome: 'continued' }]) - expect(host.isHeld(SESSION)).toBe(false) -}) - -/** Holds the continuation at send admission while `during` runs, then lets it proceed. */ -async function continueAcross( - state: Awaited>, - during: (events: NonNullable>) => void -) { - const { host, store } = state - expect(await host.restartResume.list()).toHaveLength(1) - await host.hold(SESSION, 'pane') - const events = resumedEvents(state) - if (!events) { - throw new Error('missing resumed provider event sink') - } - const admitting = Promise.withResolvers() - const proceed = Promise.withResolvers() - const admit = store.admitMutationOperation - vi.spyOn(store, 'admitMutationOperation').mockImplementationOnce(async (input) => { - admitting.resolve() - await proceed.promise - return admit(input) - }) - const continuing = host.restartResume.continueAfterRestart([SESSION], 'modal') - await admitting.promise - during(events) - await host.flushStreamedEvents(SESSION) - proceed.resolve() - return continuing -} - -function resumedEvents(state: Awaited>) { - return state.acquire.mock.calls[0]?.[0].events -} - -it('refuses at send admission once the user has sent a newer message', async () => { - const state = await interruptedRestart() - const { host, dispatch } = state - const result = await continueAcross(state, (events) => - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'newer-turn', ordinal: 1 }, - hostTestMessage('A newer task from another client') - ) - ) - expect(result.resumed).toMatchObject([ - { outcome: 'refused', reason: 'agent_session_restart_work_superseded' } - ]) - expect(dispatch).not.toHaveBeenCalled() - expect(host.journalSnapshot(SESSION).submissions).toMatchObject([ - { dispatchState: 'rejected', reason: 'agent_session_restart_work_superseded' } - ]) - host.release(SESSION, 'pane') - expect(host.isHeld(SESSION)).toBe(false) }) -// Claude opens a turn of its own on reattach to say the last session did not finish. Resume is -// pressed while that turn still runs: the continuation is sent and the provider queues it. -it('sends the continuation while a turn the provider opened is still running', async () => { - const state = await interruptedRestart() - const { host, dispatch } = state - const result = await continueAcross(state, (events) => - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'notice-turn', ordinal: 1 }, - { kind: 'turn', turnId: 'notice-turn', state: 'running' } - ) - ) - expect(result.continued).toMatchObject([{ outcome: 'continued' }]) - expect(dispatch).toHaveBeenCalledTimes(1) - host.release(SESSION, 'pane') -}) - -// The send the user made just before quitting, after an earlier exchange had finished. Neither -// that finished turn nor the provider completing the send's own turn after the restart withdraws it. +// The send the user made just before quitting, after an earlier exchange had finished: that +// finished turn does not withdraw it. it('offers and continues a send made after an earlier completed turn', async () => { - const state = await interruptedRestart('send-after-reply', false) - const { host, dispatch, marker } = state + const { host, dispatch, marker } = await interruptedRestart('send-after-reply', false) expect(marker?.work.kind).toBe('submission') - const result = await continueAcross(state, (events) => - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'original-turn', ordinal: 1 }, - { kind: 'turn', turnId: 'original-turn', state: 'completed' } - ) - ) + const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') expect(result.continued).toMatchObject([{ outcome: 'continued' }]) expect(dispatch).toHaveBeenCalledTimes(1) - host.release(SESSION, 'pane') }) -it.each([false, true])( - 'refuses a queued newer message before dispatch even if later settlement fails: %s', - async (settlementFails) => { - const { host, store, acquire, dispatch } = await interruptedRestart('submission', false) - expect(await host.restartResume.list()).toHaveLength(1) - await host.hold(SESSION, 'pane') - const events = acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') - } - const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const settle = store.recordOperationOutcome.bind(store) - let admitted = false - vi.spyOn(store, 'recordOperationOutcome').mockImplementation(async (input) => { - if (admitted) { - if (settlementFails) { - throw new Error('operation outcome could not be persisted') - } - return settle(input) - } - await settle(input) - admitted = true - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'original-turn', ordinal: 2 }, - { kind: 'status', text: 'Provider finished its last action' } - ) - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'original-turn', ordinal: 1 }, - hostTestMessage('A newer task from another client'), - { lifecycle: true } - ) - }) - - const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - - expect(result.continued).toMatchObject([ - { outcome: 'refused', reason: 'agent_session_restart_work_superseded' } - ]) - expect(dispatch).not.toHaveBeenCalled() - expect(host.journalSnapshot(SESSION).submissions).toHaveLength(2) - expect(host.journalSnapshot(SESSION).submissions[1]?.dispatchState).toBe('rejected') - host.release(SESSION, 'pane') - expect(host.isHeld(SESSION)).toBe(false) - // The offer is spent, but the refusal is kept as a durable failure: a retry finds it, and the - // superseded chat is still not eligible, so nothing runs and the record stays for the user. - expect(await host.restartResume.continueAfterRestart([SESSION], 'retry')).toMatchObject({ - resumed: [], - continued: [], - sessions: [], - failed: [ - { sessionId: SESSION, outcome: 'refused', reason: 'agent_session_restart_work_superseded' } - ] - }) - expect(dispatch).not.toHaveBeenCalled() - if (settlementFails) { - expect(store.recordOperationOutcome).toHaveBeenCalledOnce() - expect(warning).not.toHaveBeenCalled() - } - warning.mockRestore() - } -) - -// What the provider says after reattaching does not re-judge the offer. -it.each(['completed', 'approval', 'question'] as const)( - 'continues past provider %s evidence accepted while recording the continuation', - async (event) => { - const { host, acquire, dispatch } = await interruptedRestart() - await host.restartResume.list() - await host.hold(SESSION, 'pane') - const events = acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('missing resumed provider event sink') - } - const append = AgentSessionJournal.prototype.appendSubmission - const writing = vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission') - writing.mockImplementationOnce(async function (this: AgentSessionJournal, input) { - const cursor = await append.call(this, input) - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'interrupted-turn', ordinal: 1 }, - event === 'completed' - ? { kind: 'turn', turnId: 'interrupted-turn', state: 'completed' } - : { ...pendingApproval().body, question: 'Which action?', kind: event }, - { lifecycle: true } - ) - return cursor - }) - try { - const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(result.continued).toMatchObject([{ outcome: 'continued' }]) - expect(dispatch).toHaveBeenCalledTimes(1) - } finally { - writing.mockRestore() - host.release(SESSION, 'pane') - } - } -) - it('replays the same logical continuation through the durable send ledger', async () => { - const { host, store, dispatch, marker } = await interruptedRestart() - if (!marker) { - throw new Error('missing interrupted restart marker') - } + const { host, dispatch } = await interruptedRestart() + const sending = vi.spyOn(host, 'send') await host.restartResume.continueAfterRestart([SESSION], 'modal') - const fence = store.getRecord(SESSION)?.lease.runtimeFence - if (fence === undefined) { - throw new Error('missing resumed lease') + const sent = sending.mock.calls[0]?.[1] + sending.mockRestore() + if (!sent) { + throw new Error('the continuation was not sent') } const replay = await host.send( { callerKey: STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER }, - restartContinuationEnvelope(SESSION, fence, marker) + { envelope: sent.envelope, body: sent.body } ) expect(replay).toMatchObject({ ok: true, replayed: true }) - expect(host.journalSnapshot(SESSION).submissions).toHaveLength(1) + expect((await host.journalSnapshot(SESSION)).submissions).toHaveLength(1) expect(dispatch).toHaveBeenCalledTimes(1) }) +// A send that throws after Orca may have taken it is not proof it was not delivered. it.each([false, true])( - 'keeps dispatched delivery unconfirmed when settlement fails (uncertainty write fails: %s)', + 'keeps a continuation unconfirmed when its acceptance cannot be recorded (uncertainty write fails: %s)', async (uncertaintyFails) => { - const { host, store, dispatch } = await interruptedRestart() + const { host, store } = await interruptedRestart() const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) expect(await host.restartResume.list()).toHaveLength(1) - await host.hold(SESSION, 'pane') const settle = store.recordOperationOutcome.bind(store) - let dispatched = false - vi.spyOn(store, 'recordOperationOutcome').mockImplementation(async (input) => { - if (input.outcome.status === 'succeeded') { - dispatched = true - } - if (dispatched && (input.outcome.status === 'succeeded' || uncertaintyFails)) { + const recording = vi.spyOn(store, 'recordOperationOutcome') + recording.mockImplementation(async (input) => { + // Only the continuation's own record: the start it makes records its attach as usual. + if ( + input.callerKey === STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER && + (input.outcome.status === 'succeeded' || uncertaintyFails) + ) { throw new Error('operation outcome could not be persisted') } return settle(input) @@ -355,127 +155,91 @@ it.each([false, true])( const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') - expect(dispatch).toHaveBeenCalledTimes(1) - expect(host.journalSnapshot(SESSION).submissions[0]?.dispatchState).toBe('accepted') expect(result.continued).toMatchObject([{ sessionId: SESSION, outcome: 'unknown' }]) - // Filed as unconfirmed, with a warning in the chat; the continuation's own message is part of - // the filed state, so it does not retire the record it caused. + // Filed as unconfirmed, with a warning in the chat. expect(result.failed).toMatchObject([{ sessionId: SESSION, outcome: 'unconfirmed' }]) - expect(statusNotes(host)).toContainEqual({ + expect(await statusNotes(host)).toContainEqual({ text: AGENT_SESSION_RESTART_CONTINUATION_UNCONFIRMED_NOTE, tone: 'warning' }) - host.release(SESSION, 'pane') - expect(host.isHeld(SESSION)).toBe(false) - await host.restartResume.continueAfterRestart([SESSION], 'retry') - expect(dispatch).toHaveBeenCalledTimes(1) - expect(await host.restartResume.listFailures()).toMatchObject([{ outcome: 'unconfirmed' }]) - expect(warning.mock.calls.flat()).not.toContainEqual( - expect.objectContaining({ message: 'operation outcome could not be persisted' }) - ) + recording.mockRestore() warning.mockRestore() } ) -it('releases a failed acquisition and leaves the offer retryable', async () => { +// The continuation is accepted, then its start fails: the message is rejected with the cause and +// the failure is filed, and nothing is stopped because nothing started. +it('rejects the continuation when its start fails, and files a retryable refusal', async () => { const { host, acquire, dispatch, closeSession } = await interruptedRestart() acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) - expect(await host.restartResume.resume([SESSION], 'modal')).toMatchObject([ - { outcome: 'refused' } + const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') + expect(result.resumed).toMatchObject([{ outcome: 'refused' }]) + expect(result.continued).toMatchObject([{ outcome: 'refused' }]) + // Nothing ran, so the offer stands as a failure a retry can act on. + expect(result.failed).toMatchObject([{ sessionId: SESSION, outcome: 'refused', retryable: true }]) + expect(await statusNotes(host)).toContainEqual({ + text: AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE, + tone: 'error' + }) + expect((await host.journalSnapshot(SESSION)).submissions).toMatchObject([ + { dispatchState: 'rejected' } ]) - expect(host.isHeld(SESSION)).toBe(false) - await host.restartResume.continueAfterRestart([SESSION], 'retry') - expect(acquire).toHaveBeenCalledTimes(2) - expect(dispatch).toHaveBeenCalledTimes(1) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).not.toHaveBeenCalled() expect(closeSession).not.toHaveBeenCalled() }) -it.each([false, true])( - 'admits one durable continuation under concurrent calls (pane already live: %s)', - async (alreadyLive) => { - const { host, root, acquire, dispatch } = await interruptedRestart() - if (alreadyLive) { - expect(await host.restartResume.list()).toHaveLength(1) - await host.hold(SESSION, 'pane') - } - const results = await Promise.all([ - host.restartResume.continueAfterRestart([SESSION], 'window-one'), - host.restartResume.continueAfterRestart([SESSION], 'window-two') - ]) - expect( - results.flatMap((result) => result.resumed).filter((r) => r.outcome === 'resumed') - ).toHaveLength(1) - expect( - results.flatMap((result) => result.continued).filter((r) => r.outcome === 'continued') - ).toHaveLength(1) - expect(acquire).toHaveBeenCalledTimes(1) - expect(dispatch).toHaveBeenCalledTimes(1) - expect(host.journalSnapshot(SESSION).submissions).toHaveLength(1) - expect(await new AgentSessionRecoveryCapsule(root).list(NOW)).toEqual([]) - await host.restartResume.continueAfterRestart([SESSION], 'later-click') - expect(dispatch).toHaveBeenCalledTimes(1) - host.release(SESSION, 'pane') - } -) - -it.each([false, true])( - 'releases reconnect acquisition to idle eviction (pane: %s)', - async (pane) => { - const { host, acquire, dispatch, closeSession } = await interruptedRestart() - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - expect(await host.restartResume.resume([SESSION], 'modal')).toMatchObject([ - { outcome: 'resumed' } - ]) - expect(host.isHeld(SESSION)).toBe(false) - if (pane) { - await host.hold(SESSION, 'pane') - await vi.advanceTimersByTimeAsync(GRACE * 2) - expect(closeSession).not.toHaveBeenCalled() - host.release(SESSION, 'pane') - } - await vi.advanceTimersByTimeAsync(GRACE) - await vi.waitFor(() => expect(closeSession).toHaveBeenCalledTimes(1)) - expect(acquire).toHaveBeenCalledTimes(1) - expect(dispatch).not.toHaveBeenCalled() - } -) +it('admits one durable continuation under concurrent calls', async () => { + const { host, root, acquire, dispatch } = await interruptedRestart() + const results = await Promise.all([ + host.restartResume.continueAfterRestart([SESSION], 'window-one'), + host.restartResume.continueAfterRestart([SESSION], 'window-two') + ]) + expect( + results.flatMap((result) => result.resumed).filter((r) => r.outcome === 'resumed') + ).toHaveLength(1) + expect( + results.flatMap((result) => result.continued).filter((r) => r.outcome === 'continued') + ).toHaveLength(1) + expect(acquire).toHaveBeenCalledTimes(1) + expect(dispatch).toHaveBeenCalledTimes(1) + expect((await host.journalSnapshot(SESSION)).submissions).toHaveLength(1) + expect(await new AgentSessionRecoveryCapsule(root).list(NOW)).toEqual([]) + await host.restartResume.continueAfterRestart([SESSION], 'later-click') + expect(dispatch).toHaveBeenCalledTimes(1) +}) -it('retains acquisition through slow continuation settlement, then releases it', async () => { - const { host, dispatch, closeSession } = await interruptedRestart() - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) +// The batch counts a chat done once its agent took the continuation; the provider's slow answer is +// awaited after, and decides the outcome. +it('waits out a slow provider answer before reporting the continuation', async () => { + const { host, dispatch } = await interruptedRestart() const settlement = Promise.withResolvers>>() const dispatched = Promise.withResolvers() dispatch.mockImplementationOnce(() => { dispatched.resolve() return settlement.promise }) - const continuing = host.restartResume.continueAfterRestart([SESSION], 'modal') + let settled = false + const continuing = host.restartResume + .continueAfterRestart([SESSION], 'modal') + .finally(() => (settled = true)) await dispatched.promise - await vi.advanceTimersByTimeAsync(GRACE * 2) - expect(host.isHeld(SESSION)).toBe(true) - expect(closeSession).not.toHaveBeenCalled() + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(settled).toBe(false) settlement.resolve({ state: 'rejected', reason: 'provider refused' }) expect((await continuing).continued).toMatchObject([{ outcome: 'refused' }]) - expect(host.isHeld(SESSION)).toBe(false) - await vi.advanceTimersByTimeAsync(GRACE) - await vi.waitFor(() => expect(closeSession).toHaveBeenCalledTimes(1)) expect(dispatch).toHaveBeenCalledTimes(1) }) -// Opening the chat is inspection only. The explicit restart action is what removes the durable -// offer, so ordinary pane lifecycle must not make this status disappear. +// Opening the chat is inspection only: it starts nothing, and the explicit restart action is what +// removes the durable offer, so reading the chat must not make this status disappear. it('keeps offering a chat after the user opens it', async () => { - const { host, root, store } = await interruptedRestart() + const { host, root, acquire } = await interruptedRestart() expect(await host.restartResume.list()).toHaveLength(1) - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - await host.hold(SESSION, 'pane') - host.release(SESSION, 'pane') - await vi.advanceTimersByTimeAsync(GRACE) - // The whole eviction, not just the provider stop: the lease returns to `released` on the step - // before the last, and until it does the offer is refused for a reason that is not recovery. - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - vi.useRealTimers() + const unsubscribe = await host.subscribe({ id: 'pane', sessionId: SESSION, emit: vi.fn() }) + await host.history({ sessionId: SESSION, direction: 'tail' }) + unsubscribe() + expect(acquire).not.toHaveBeenCalled() expect(await host.restartResume.list()).toHaveLength(1) await host.restartResume.recordMarkers() @@ -544,7 +308,7 @@ it('keeps concurrent recovery reads independent and non-destructive', async () = list.mockRestore() }) -it('fails closed on corrupt recovery storage while ordinary hold and send still work', async () => { +it('fails closed on corrupt recovery storage while an ordinary send still works', async () => { const { host, root, dispatch } = await interruptedRestart() await writeFile(join(root, AGENT_SESSION_RECOVERY_CAPSULE_FILE), '{') const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) @@ -555,81 +319,56 @@ it('fails closed on corrupt recovery storage while ordinary hold and send still sessions: [], failed: [] }) - await host.hold(SESSION, 'pane') const body = hostTestMessage('A fresh ordinary request') expect( await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) ).toMatchObject({ ok: true }) - expect(dispatch).toHaveBeenCalledTimes(1) - // list; the action's read of offers and of failures; the post-action refresh of both. - expect(warning).toHaveBeenCalledTimes(5) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + // list; the action's read of offers and of failures; the post-action refresh of both. The send + // cannot withdraw an offer it cannot read either, and says so. + const withdrawing = '[structured-agent-session] withdrawing a restart offer failed' + await vi.waitFor(() => expect(warning).toHaveBeenLastCalledWith(withdrawing)) + expect(warning.mock.calls.filter(([message]) => message !== withdrawing)).toHaveLength(5) warning.mockRestore() - host.release(SESSION, 'pane') }) -// The toast is gone in seconds and the offer is spent by the reattach, so without this record -// nothing on any surface would still name the chat the user has to continue by hand. -it('keeps a refused continuation as a durable failure that names the chat and the reason', async () => { - const { host, root, result } = await supersededRefusal() - const failure = { - sessionId: SESSION, - outcome: 'refused', - reason: 'agent_session_restart_work_superseded', - latestPrompt: expect.any(String), - agent: 'codex', - retryable: false - } - expect(result).toMatchObject({ sessions: [], failed: [failure] }) - // The chat itself says what happened and what to do. - expect(statusNotes(host)).toContainEqual({ - text: AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE, - tone: 'error' +// The user's own message was accepted first: the continuation is refused, and since nothing the +// user did failed, the chat says nothing and no failure is kept. +it("refuses a continuation quietly when the user's own message was accepted first", async () => { + const { host, root, dispatch, result } = await supersededRefusal() + expect(result).toMatchObject({ + continued: [{ outcome: 'refused', reason: 'agent_session_restart_work_superseded' }], + sessions: [], + failed: [] }) - expect(await host.restartResume.list()).toEqual([]) - expect(await host.restartResume.listFailures()).toMatchObject([failure]) - // Durable: a fresh reader of the same file sees it too. - expect(await new AgentSessionRecoveryCapsule(root).listFailed(NOW)).toMatchObject([ - { marker: { sessionId: SESSION }, outcome: 'refused' } - ]) - host.release(SESSION, 'pane') + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) + expect(await statusNotes(host)).toEqual([]) + expect(await new AgentSessionRecoveryCapsule(root).list(NOW)).toEqual([]) + expect(await new AgentSessionRecoveryCapsule(root).listFailed(NOW)).toEqual([]) }) -// The failure asked the user to continue the chat themselves; their own message is that -// continuation. Nothing on the send path clears it: the listing sees the newer message. +/** A continuation whose start failed, filed as a retryable failure. */ +async function failedContinuation() { + const state = await interruptedRestart() + state.acquire.mockRejectedValueOnce(new Error('provider could not reconnect')) + const result = await state.host.restartResume.continueAfterRestart([SESSION], 'modal') + expect(result.failed).toMatchObject([{ sessionId: SESSION, outcome: 'refused' }]) + return state +} + +// Starting the chat's agent again answers a failure, as it ends an offer. it('retires a recorded failure once the user sends in that chat, with no send hook', async () => { - const { host, root } = await supersededRefusal() + const { host, root, dispatch } = await failedContinuation() const body = hostTestMessage('Carry on from where you stopped') await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - expect(await host.restartResume.listFailures()).toEqual([]) - // Pruned from the file too, not only hidden. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) + // Pruned from the file, not only hidden. await vi.waitFor(async () => { expect(await new AgentSessionRecoveryCapsule(root).listFailed(NOW)).toEqual([]) }) - host.release(SESSION, 'pane') + expect(await host.restartResume.listFailures()).toEqual([]) }) -// The user can reply in a chat while other chats in the same action are still being continued, -// before its turn in the batch or after its own note asks them to. Either reply answers the failure. -it.each(['before', 'after'] as const)( - 'retires a failure the user answered %s its own attempt, before the action settled', - async (userAnswers) => { - const { host, root, result } = await supersededRefusal(userAnswers) - expect(result.resumed).toMatchObject([ - userAnswers === 'before' ? { reason: 'agent_session_resume_not_eligible' } : {} - ]) - expect( - statusNotes(host).some( - (note) => note.text === AGENT_SESSION_RESTART_CONTINUATION_REFUSED_NOTE - ) - ).toBe(userAnswers === 'after') - expect(result.failed).toEqual([]) - await vi.waitFor(async () => { - expect(await new AgentSessionRecoveryCapsule(root).listFailed(NOW)).toEqual([]) - }) - host.release(SESSION, 'pane') - } -) - it('removes a failure when a named retry succeeds', async () => { const { host, root, dispatch } = await interruptedRestart() const capsule = new AgentSessionRecoveryCapsule(root) @@ -645,7 +384,7 @@ it('removes a failure when a named retry succeeds', async () => { outcome: 'refused', reason: 'agent_session_conflict', latestPrompt: '', - latestUserItemId: pending!.latestUserItemId + latestUserItemId: pending!.latestUserItemId ?? null } ], NOW @@ -661,7 +400,7 @@ it('removes a failure when a named retry succeeds', async () => { }) it('dismisses one failure by name and leaves the rest of the durable records alone', async () => { - const { host, root } = await supersededRefusal() + const { host, root } = await failedContinuation() const capsule = new AgentSessionRecoveryCapsule(root) const other = parseAgentSessionResumeMarker({ sessionId: 'session-other', @@ -680,7 +419,6 @@ it('dismisses one failure by name and leaves the rest of the durable records alo expect(await host.restartResume.dismiss([SESSION])).toBe(1) expect(await host.restartResume.listFailures()).toEqual([]) expect(await capsule.list(NOW)).toEqual([other]) - host.release(SESSION, 'pane') }) it('logs teardown capsule publication failure and still releases the provider', async () => { @@ -699,7 +437,9 @@ it('logs teardown capsule publication failure and still releases the provider', await mkdir(capsulePath) const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) await expect(previous.host.flushAllStreamedEvents()).resolves.toBeUndefined() - expect(() => previous.host.journalSnapshot(SESSION)).toThrow('agent_session_ownership_unknown') + await expect(previous.host.journalSnapshot(SESSION)).rejects.toThrow( + 'agent_session_ownership_unknown' + ) expect(warning).toHaveBeenCalledWith( '[structured-agent-session] recording recovery capsule failed' ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index 3b6cc30271e5..b4a54ea0726e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -20,6 +20,8 @@ export function createRestartReconciler(deps: { records: readonly AgentSessionRecord[] ) => Promise> now: () => number + /** A run that settled every lease; what it flagged is owed settlement now. */ + onReconciled?: () => void }): (sessionId: string) => Promise { let pending: Promise | null = null return async (sessionId) => { @@ -27,7 +29,7 @@ export function createRestartReconciler(deps: { return null } if (!pending) { - const run = reconcileCurrentLeases(deps) + const run = reconcileCurrentLeases(deps).then(() => deps.onReconciled?.()) pending = run.finally(() => { pending = null }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore-gate.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore-gate.test.ts deleted file mode 100644 index 4567d84d5c0d..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore-gate.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' - -describe('StructuredAgentSessionRestartRestoreGate', () => { - it('runs one successful restore across concurrent and later discovery', async () => { - const gate = new StructuredAgentSessionRestartRestoreGate() - const restore = vi.fn(async () => undefined) - - await Promise.all([gate.run(restore), gate.run(restore), gate.run(restore)]) - await gate.run(restore) - - expect(restore).toHaveBeenCalledOnce() - }) - - it('allows a failed restore to be retried', async () => { - const gate = new StructuredAgentSessionRestartRestoreGate() - const restore = vi - .fn<() => Promise>() - .mockRejectedValueOnce(new Error('restore failed')) - .mockResolvedValue(undefined) - - await expect(gate.run(restore)).rejects.toThrow('restore failed') - await expect(gate.run(restore)).resolves.toBeUndefined() - - expect(restore).toHaveBeenCalledTimes(2) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore-gate.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore-gate.ts deleted file mode 100644 index dd4237dcb28b..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore-gate.ts +++ /dev/null @@ -1,17 +0,0 @@ -export class StructuredAgentSessionRestartRestoreGate { - private current: Promise | null = null - - run(restore: () => Promise): Promise { - if (this.current) { - return this.current - } - const tracked = restore().catch((error: unknown) => { - if (this.current === tracked) { - this.current = null - } - throw error - }) - this.current = tracked - return tracked - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts deleted file mode 100644 index f46ff48ce15b..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionAttachParams } from './structured-agent-session-attach' - -const { restoreRead } = vi.hoisted(() => ({ - restoreRead: vi.fn() -})) - -vi.mock('./structured-agent-session-read-restore', () => ({ - restoreStructuredAgentSessionRead: restoreRead -})) - -import { - restoreOneStructuredAgentSessionRead, - restoreStructuredAgentSessionsOnRestart -} from './structured-agent-session-restart-restore' - -describe('restart journal restoration', () => { - beforeEach(() => restoreRead.mockReset()) - - it('bounds historical journal parsing to four sessions at a time', async () => { - const gate = Promise.withResolvers() - let active = 0 - let peak = 0 - restoreRead.mockImplementation(async (_store, _root, sessionId: string) => { - active += 1 - peak = Math.max(peak, active) - await gate.promise - active -= 1 - return { - journal: {}, - params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' }, - fence: 1, - hasProviderChild: false, - sessionId - } - }) - const records = Array.from( - { length: 12 }, - (_, index) => ({ sessionId: `session-${index}` }) as AgentSessionRecord - ) - - const restoration = restoreStructuredAgentSessionsOnRestart({ - store: {} as never, - journalRoot: '/tmp/journals', - records, - reconcile: async () => null, - resolveRecovery: async () => undefined, - serialize: async (_sessionId, task) => task(), - hasSession: () => false, - onReadable: () => undefined, - retrySettlement: async () => true - }) - - await vi.waitFor(() => expect(active).toBe(4)) - expect(restoreRead).toHaveBeenCalledTimes(4) - gate.resolve() - await restoration - - expect(restoreRead).toHaveBeenCalledTimes(records.length) - expect(peak).toBe(4) - }) - - it('runs pending settlement retry after recovery resolution', async () => { - const calls: string[] = [] - const params: AgentSessionAttachParams = { - envelope: { - sessionId: 'session-1', - clientOperationId: 'read-restore:session-1', - expectedRuntimeFence: 4, - payloadFingerprint: 'fingerprint' - }, - location: { - executionHostId: 'local', - wslDistro: null, - workspaceId: 'workspace-1', - workspaceKind: 'folder' - }, - provider: 'codex', - agent: 'codex', - accountHome: { variable: 'CODEX_HOME', path: '/tmp/codex' }, - runtimeKind: 'native' - } - restoreRead.mockResolvedValue({ - journal: {}, - params, - fence: 4, - hasProviderChild: false, - acquisitionGeneration: null - }) - - await restoreOneStructuredAgentSessionRead( - { - store: {} as never, - journalRoot: '/tmp/journals', - reconcile: async () => null, - resolveRecovery: async () => { - calls.push('resolveRecovery') - }, - serialize: async (_sessionId, task) => task(), - hasSession: () => false, - onReadable: () => { - calls.push('onReadable') - }, - retrySettlement: async (_sessionId, restoredParams) => { - calls.push( - restoredParams === params ? 'retrySettlement:restored-params' : 'retrySettlement' - ) - return true - } - }, - 'session-1' - ) - - expect(calls).toEqual(['resolveRecovery', 'onReadable', 'retrySettlement:restored-params']) - }) - - it('does not rerun settlement retry when a second restore finds the session already open', async () => { - const retrySettlement = vi.fn(async () => true) - restoreRead.mockResolvedValue({ - journal: {}, - params: {}, - fence: 4, - hasProviderChild: false, - acquisitionGeneration: null - }) - - await restoreOneStructuredAgentSessionRead( - { - store: {} as never, - journalRoot: '/tmp/journals', - reconcile: async () => null, - resolveRecovery: async () => undefined, - serialize: async (_sessionId, task) => task(), - hasSession: () => true, - onReadable: () => undefined, - retrySettlement - }, - 'session-1' - ) - - expect(retrySettlement).not.toHaveBeenCalled() - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 38c04d2a5973..5a431e3f62bb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -2,90 +2,360 @@ // // It owes reconciliation — every lease loaded from disk names an owner from a process generation // that no longer exists, and adjudicating that is startup's job. It owes an exit from any recovery -// stage the evidence now permits. And it owes a READABLE session: the journal open, history -// answerable, the tab restorable. +// stage the evidence now permits, a settlement for any turn a restart cut off, and a status row for +// every listed chat. It does not owe a provider child, and it does not owe an open journal: a chat +// whose saved status still matches its journal's position is listed from that copy, and its journal +// opens when something reads it. // -// It does not owe a provider child. This used to resume every record whose lease was `released`, -// which is the normal end state of a chat the user closed cleanly — so a -// healthy profile started an app-server per session it had ever used, in parallel, at every launch, -// with no client attached and nothing on screen. A child now exists because a surface asked for the -// session (see `structured-agent-session-holds`), not because a record survived on disk. +// One pass, kicked once after the host installs, in two phases: +// 1. before reconcile, active workspace first: each listed chat's saved status is checked against +// its journal's position; a match publishes the row, anything else is remembered as a miss; +// 2. after reconcile: recovery exits, then settlement of every open conversation, then each miss +// and each listed record still owing settlement is opened (unless already open), settled, and +// closed again when closable. A startup reconcile that fails leaves that half owed: the first +// clean reconcile after it, a bounded retry's or anyone's, runs it once more. +// Every step runs under its own session's lock with a yield between chats, so a read or a send +// jumps the queue and waits for one chat at most. +import { existsSync } from 'node:fs' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { yieldToEventLoop } from '../../../shared/event-loop-yield' +import { + STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, + type StructuredAgentSessionSavedStatus +} from '../../../shared/structured-agent-session-saved-status' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { mapWithConcurrency } from '../../../shared/map-with-concurrency' +import { sameJournalCursor } from '../agent-session-journal/journal-cursor' +import { probeJournalCursor } from '../agent-session-journal/journal-cursor-probe' +import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant' +import { journalDatabaseFile, journalDirectoryFor } from '../agent-session-journal/journal-paths' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' +import { adapterSupportsRecord } from './structured-agent-session-provider-support' +import { structuredSessionRecoveryIsResolvable } from './structured-agent-session-recovery-resolution' import { - restoreStructuredAgentSessionRead, - type RestoredStructuredAgentSessionRead -} from './structured-agent-session-read-restore' + retryLoadedStructuredAgentSessionSettlement, + structuredAgentSessionOwesSettlement +} from './structured-agent-session-settlement-retry' + +/** Which chats a user is most likely to look at first; absent on hosts with no saved desktop + * session (`orca serve`, SSH), where the pass keeps the listing's order. */ +export type StructuredAgentSessionStartupPriority = { + activeWorkspaceId: string | null + visibleWorkspaceIds: ReadonlySet +} + +/** One chat's share of the pass, run under its lock. `session` is the conversation open when the + * step began; `close` drops it again by the idle sweep's rule, so one still owing work stays. */ +export type StructuredAgentSessionStartupStep = ( + sessionId: string, + step: (context: { + session: StructuredAgentSessionHostSession | undefined + close: () => Promise + }) => Promise +) => Promise -const JOURNAL_RESTORE_CONCURRENCY = 4 +/** Waits before each retry of a startup reconcile that did not complete. */ +const STARTUP_RECONCILE_RETRY_DELAYS_MS = [1_000, 4_000, 15_000] -export type StructuredAgentSessionReadRestoreDeps = { - store: AgentSessionRecordStore +export type StructuredAgentSessionStartupPassDeps = { + store: Pick journalRoot: string - reconcile: (sessionId: string) => Promise + savedStatus?: { + read: (sessionId: string) => StructuredAgentSessionSavedStatus | null + prune: (keep: (sessionId: string) => boolean) => void + } + supportsRecord: (record: AgentSessionRecord) => boolean + sessions: ReadonlyMap + step: StructuredAgentSessionStartupStep + seed: (sessionId: string, saved: StructuredAgentSessionSavedStatus) => void + reconcile: () => Promise resolveRecovery: (sessionId: string) => Promise - serialize: (sessionId: string, task: () => Promise) => Promise - hasSession: (sessionId: string) => boolean - onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void - retrySettlement: ( - sessionId: string, - params: RestoredStructuredAgentSessionRead['params'] - ) => Promise + /** The host's one open, for a caller inside the session's lock; it re-checks for an open entry. */ + open: (sessionId: string) => Promise + settle: (sessionId: string, journal: AgentSessionJournal) => Promise + onError: (sessionId: string, error: unknown) => void + now: () => number + /** Quit has begun: no retry starts after this. */ + disposed: () => boolean + reconcileRetryDelaysMs?: readonly number[] | undefined } -/** - * One session's share of the restart restore, and the whole of an on-demand one. - * - * Startup maps this over every supported record; a surface asking for a session it cannot see - * calls it for one id. The CALLER decides which records are eligible — startup filters by - * `supportsRecord` before mapping, so an on-demand caller owes the same check. - */ -export async function restoreOneStructuredAgentSessionRead( - input: StructuredAgentSessionReadRestoreDeps, - sessionId: string -): Promise { - const unreconciled = await input.reconcile(sessionId) - if (!unreconciled) { - // A session latched in recovery exits here at startup, without waiting for a client. - await input.resolveRecovery(sessionId) - } - await input.serialize(sessionId, () => - restoreOneStructuredAgentSessionReadUnderSerialize(input, sessionId) +export type StructuredAgentSessionStartupPass = { + /** Latched: every later call is the same pass, retried only after a failure. */ + run: (priority?: StructuredAgentSessionStartupPriority) => Promise + /** A later reconcile settled every lease: settle what it flagged on open conversations, or, after + * a failed startup reconcile, run the whole post-reconcile half once. */ + onReconciled: () => void +} + +/** A compaction or rewind the last run left prepared: the chat's open settles it and writes a row, + * so its saved status cannot equal what that open publishes. */ +function openOwesCommandSettlement(record: AgentSessionRecord): boolean { + const command = record.conversationCommand + return ( + (command?.command === 'compact' && command.phase === 'prepared') || + record.rewind?.phase === 'prepared' || + record.rewind?.phase === 'provider-succeeded' ) } -/** The serialized half of the restore, for a caller already inside the session's serialize — a - * send replaying into a session this host has closed, which needs the journal and no child. */ -export async function restoreOneStructuredAgentSessionReadUnderSerialize( - input: Pick< - StructuredAgentSessionReadRestoreDeps, - 'store' | 'journalRoot' | 'hasSession' | 'onReadable' | 'retrySettlement' - >, - sessionId: string -): Promise { - if (input.hasSession(sessionId)) { - // A surface that took a hold mid-restore already attached this one. - return +function journalDirFor(root: string, record: AgentSessionRecord): string { + return journalDirectoryFor(root, { + workspaceId: record.location.workspaceId, + sessionId: record.sessionId + }) +} + +/** Active workspace first, then the other visible ones, then the rest; listing order within each. */ +export function orderStartupSessionIds( + sessionIds: readonly string[], + workspaceOf: (sessionId: string) => string | undefined, + priority: StructuredAgentSessionStartupPriority | undefined +): string[] { + if (!priority) { + return [...sessionIds] } - const restored = await restoreStructuredAgentSessionRead( - input.store, - input.journalRoot, - sessionId - ) - if (!restored) { - return + const tier = (sessionId: string): number => { + const workspaceId = workspaceOf(sessionId) + if (workspaceId !== undefined && workspaceId === priority.activeWorkspaceId) { + return 0 + } + return workspaceId !== undefined && priority.visibleWorkspaceIds.has(workspaceId) ? 1 : 2 } - input.onReadable(sessionId, restored) - await input.retrySettlement(sessionId, restored.params) + return sessionIds + .map((sessionId, index) => ({ sessionId, index, tier: tier(sessionId) })) + .sort((left, right) => left.tier - right.tier || left.index - right.index) + .map(({ sessionId }) => sessionId) } -export async function restoreStructuredAgentSessionsOnRestart( - input: StructuredAgentSessionReadRestoreDeps & { records: AgentSessionRecord[] } -): Promise { - await mapWithConcurrency(input.records, JOURNAL_RESTORE_CONCURRENCY, ({ sessionId }) => - restoreOneStructuredAgentSessionRead(input, sessionId) - ) +export function createStructuredAgentSessionStartupPass( + deps: StructuredAgentSessionStartupPassDeps +): StructuredAgentSessionStartupPass { + let running: Promise | null = null + let settledOpenOnce = false + let priority: StructuredAgentSessionStartupPriority | undefined + // A failed startup reconcile left recovery and settlement owed until a clean one lands. + let startupReconcileFailed = false + let cleanReconcileSinceFailure = false + let afterFailure: Promise | null = null + const listed = (sessionId: string): boolean => + deps.store.listVisibleSessionIds().includes(sessionId) + const workspaceOf = (sessionId: string) => deps.store.getRecord(sessionId)?.location.workspaceId + const inOrder = (ids: readonly string[]) => orderStartupSessionIds(ids, workspaceOf, priority) + const supportedRecord = (sessionId: string): AgentSessionRecord | null => { + const record = deps.store.getRecord(sessionId) + return record && deps.supportsRecord(record) ? record : null + } + + /** Phase 1's step: true when the saved copy was proven current and published. */ + const seedFromSavedStatus = (record: AgentSessionRecord): boolean => { + let saved: StructuredAgentSessionSavedStatus | null = null + try { + saved = deps.savedStatus?.read(record.sessionId) ?? null + } catch (error) { + // A saved copy that cannot be read is a miss, never a lost chat. + deps.onError(record.sessionId, error) + } + if ( + !saved || + saved.v !== STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION || + openOwesCommandSettlement(record) + ) { + return false + } + const position = probeJournalCursor(journalDirFor(deps.journalRoot, record), record.sessionId) + if (!position || !sameJournalCursor(saved.cursor, position)) { + return false + } + deps.seed(record.sessionId, saved) + return true + } + + const settleOpenConversations = async (): Promise => { + for (const sessionId of deps.sessions.keys()) { + await deps + .step(sessionId, async ({ session }) => { + if (session) { + await deps.settle(sessionId, session.journal) + } + }) + .catch((error: unknown) => deps.onError(sessionId, error)) + } + } + + /** Phase 2.3's step: open unless open, always settle, and close what only this pass opened. */ + const openAndSettle = async (sessionId: string): Promise => { + await deps.step(sessionId, async ({ session, close }) => { + const record = supportedRecord(sessionId) + if (!record || !listed(sessionId)) { + return + } + if (!session) { + const dir = journalDirFor(deps.journalRoot, record) + // No rows means no turn, and a chat with no turn has no row to publish. + if (!existsSync(journalDatabaseFile(dir)) && !findJournalFileFormatRemnant(dir)) { + return + } + } + const open = session ?? (await deps.open(sessionId)) + if (!open) { + return + } + await deps.settle(sessionId, open.journal) + if (!session) { + await close() + } + }) + } + + /** Everything a restart owes once reconcile has answered; `reconciled` gates recovery exits. */ + const settleAfterReconcile = async ( + reconciled: boolean, + misses: readonly string[] + ): Promise => { + if (reconciled) { + // Every record, hidden ones too: an orphaned owner of a closed tab is still this host's. + for (const record of deps.store.listRecords()) { + if (structuredSessionRecoveryIsResolvable(record)) { + await deps + .resolveRecovery(record.sessionId) + .catch((error: unknown) => deps.onError(record.sessionId, error)) + } + } + } + await settleOpenConversations() + settledOpenOnce = true + const owed = deps.store + .listVisibleSessionIds() + .filter((sessionId) => structuredAgentSessionOwesSettlement(deps.store.getRecord(sessionId))) + for (const sessionId of inOrder([...new Set([...misses, ...owed])])) { + // Logged once, never retried by the pass: the first read reports it. + await openAndSettle(sessionId).catch((error: unknown) => deps.onError(sessionId, error)) + await yieldToEventLoop() + } + } + + /** At most once per failed startup: the misses were already opened, so only what reconcile + * flagged is left. */ + const settleAfterFailedStartup = (): Promise => { + afterFailure ??= settleAfterReconcile(true, []).catch((error: unknown) => + deps.onError('startup-pass', error) + ) + return afterFailure + } + + const reconcileClean = (): Promise => + deps.reconcile().then( + (refusal) => refusal === null, + (error: unknown) => { + deps.onError('startup-pass', error) + return false + } + ) + + /** Retries the startup reconcile a few times, then leaves the rest to each chat's own attach. */ + const retryStartupReconcile = async (): Promise => { + for (const delay of deps.reconcileRetryDelaysMs ?? STARTUP_RECONCILE_RETRY_DELAYS_MS) { + await new Promise((resolve) => setTimeout(resolve, delay).unref?.()) + if (deps.disposed() || afterFailure) { + return + } + if (await reconcileClean()) { + await settleAfterFailedStartup() + return + } + } + deps.onError('startup-pass', new Error('startup reconcile still failing after its retries')) + } + + const runOnce = async (): Promise => { + const startedAt = deps.now() + const misses: string[] = [] + let seeded = 0 + for (const sessionId of inOrder(deps.store.listVisibleSessionIds())) { + await deps + .step(sessionId, async ({ session }) => { + const record = supportedRecord(sessionId) + if (session || !record || !listed(sessionId)) { + return + } + if (seedFromSavedStatus(record)) { + seeded += 1 + } else { + misses.push(sessionId) + } + }) + .catch((error: unknown) => deps.onError(sessionId, error)) + await yieldToEventLoop() + } + try { + deps.savedStatus?.prune((sessionId) => deps.store.getRecord(sessionId) !== null) + } catch (error) { + deps.onError('startup-pass', error) + } + console.info( + `[structured-agent-session] startup status: ${seeded} from saved copies, ${misses.length} to open, ${deps.now() - startedAt} ms` + ) + // Settled either way: a failed reconcile still leaves the misses to open and publish. + const reconciled = await reconcileClean() + startupReconcileFailed = !reconciled + await settleAfterReconcile(reconciled, misses) + if (reconciled) { + return + } + if (cleanReconcileSinceFailure) { + await settleAfterFailedStartup() + return + } + void retryStartupReconcile() + } + + return { + run: (startupPriority) => { + priority ??= startupPriority + running ??= runOnce().catch((error: unknown) => { + running = null + throw error + }) + return running + }, + onReconciled: () => { + if (startupReconcileFailed && !afterFailure) { + if (settledOpenOnce) { + void settleAfterFailedStartup() + } else { + // The failed startup's own half is still settling; it runs the rest when it finishes. + cleanReconcileSinceFailure = true + } + } else if (settledOpenOnce) { + void settleOpenConversations() + } + } + } +} + +/** The pass over a host's own deps: its store, journals, saved statuses and settlement. */ +export function createStructuredAgentSessionHostStartupPass( + deps: StructuredAgentSessionHostDeps, + wiring: Pick< + StructuredAgentSessionStartupPassDeps, + 'sessions' | 'step' | 'seed' | 'reconcile' | 'resolveRecovery' | 'open' | 'now' | 'disposed' + > +): StructuredAgentSessionStartupPass { + return createStructuredAgentSessionStartupPass({ + ...wiring, + store: deps.store, + journalRoot: deps.journalRoot, + ...(deps.savedStatus ? { savedStatus: deps.savedStatus } : {}), + reconcileRetryDelaysMs: deps.startupReconcileRetryDelaysMs, + supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), + settle: (sessionId, journal) => + retryLoadedStructuredAgentSessionSettlement({ deps, sessionId, journal, now: wiring.now }), + onError: (sessionId, error) => deps.onEventSinkError?.({ sessionId, error }) + }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts index 29453e74652b..058f6bc7c957 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts @@ -1,6 +1,6 @@ // Restart offers are durable per-session records. Listing reserves nothing and removes only offers // the chat has provably moved past; an explicit action reserves records, and only a completed -// action removes them — or files what went wrong. +// action removes them — or files what went wrong. Starting the chat's agent again withdraws them. import { randomUUID } from 'node:crypto' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' @@ -9,13 +9,6 @@ import type { AgentSessionResumeMarker, AgentSessionResumeTrigger } from '../../../shared/agent-session-resume-marker' -import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' -import type { - AgentSessionMutationEnvelope, - AgentSessionMutationResult, - AgentSessionSendResult -} from '../../../shared/agent-session-wire' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { createStructuredAgentSessionRestartCandidateReader } from './structured-agent-session-restart-candidates' import { @@ -34,32 +27,21 @@ import { type StructuredAgentSessionResumeOutcome } from './structured-agent-session-restart-resume-runner' import { - continueStructuredAgentSessionAfterRestart, - noteRestartReattachFailed, restartContinuationDeps, + startStructuredAgentSessionContinuation, type StructuredAgentSessionContinuationHost, type StructuredAgentSessionContinuationOutcome } from './structured-agent-session-restart-continuation' +import { restartContinuationId } from './structured-agent-session-restart-continuation-envelope' +import { + createStructuredAgentSessionRestartOfferWithdrawal, + type StructuredAgentSessionRestartOfferSession +} from './structured-agent-session-restart-offer-withdrawal' +import type { StructuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' import { createStructuredAgentSessionRestartWitnesses } from './structured-agent-session-restart-witnesses' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' -type LiveSession = { journal: AgentSessionJournal; hasProviderChild: boolean; fence: number } - -export type StructuredAgentSessionRestartResumeSurfaces = { - revealSession: (sessionId: string) => Promise<{ readable: boolean }> - hold: (sessionId: string, holderId: string) => Promise - release: (sessionId: string, holderId: string) => void - send: (input: { - envelope: AgentSessionMutationEnvelope - body: AgentJournalMessageItem - beforeRun?: () => void - }) => Promise> - awaitSendSettlement: ( - sessionId: string, - clientMessageId: string - ) => Promise<{ value: AgentSessionSendResult } | undefined> - onNoteFailed: (sessionId: string, error: unknown) => void - now: () => number -} +type LiveSession = StructuredAgentSessionRestartOfferSession export type StructuredAgentSessionRestartResume = { /** Teardown: begin, then per session a snapshot right before its child stops and a confirmation @@ -71,10 +53,6 @@ export type StructuredAgentSessionRestartResume = { list: () => Promise /** Offers already acted on whose agent did not carry on. Read-only; nothing here is spent. */ listFailures: () => Promise - resume: ( - sessionIds: readonly string[] | undefined, - owner: string - ) => Promise continueAfterRestart: ( sessionIds: readonly string[] | undefined, owner: string @@ -86,6 +64,8 @@ export type StructuredAgentSessionRestartResume = { }> /** Named sessions forget their offer or failure; unnamed, every durable record goes. */ dismiss: (sessionIds?: readonly string[]) => Promise + /** The chat's agent proved a start: its offer ends unless the start is a resume's own. */ + onAgentStarted: (sessionId: string) => void } export function createStructuredAgentSessionRestartResume( @@ -100,11 +80,6 @@ export function createStructuredAgentSessionRestartResume( const admission = new StructuredAgentSessionResumeAdmission() const enqueueRecoveryOperation = createStructuredAgentSessionRestartOperationQueue() - const derive = createStructuredAgentSessionRestartCandidateReader({ - sessions, - getRecord: deps.store.getRecord, - adapter: deps.adapter - }) const witnesses = createStructuredAgentSessionRestartWitnesses({ sessions, getRecord: deps.store.getRecord, @@ -114,15 +89,24 @@ export function createStructuredAgentSessionRestartResume( now: surfaces.now, enqueue: enqueueRecoveryOperation }) - const failures = createStructuredAgentSessionRestartFailureLedger({ + const withdrawal = createStructuredAgentSessionRestartOfferWithdrawal({ + sessions, ...(deps.recoveryCapsule ? { capsule: deps.recoveryCapsule } : {}), + now: surfaces.now, + enqueue: enqueueRecoveryOperation + }) + const derive = createStructuredAgentSessionRestartCandidateReader({ sessions, - reveal: async (sessionId) => { - await surfaces.revealSession(sessionId).catch(() => null) - }, + getRecord: deps.store.getRecord, + adapter: deps.adapter, + movedOn: withdrawal.movedOn + }) + const failures = createStructuredAgentSessionRestartFailureLedger({ + ...(deps.recoveryCapsule ? { capsule: deps.recoveryCapsule } : {}), getRecord: deps.store.getRecord, adapter: deps.adapter, retryable: (marker) => derive([marker], 'may-be-held').candidates.length === 1, + reveal: (markers) => revealMarkers(markers), now: surfaces.now, enqueue: enqueueRecoveryOperation }) @@ -152,19 +136,20 @@ export function createStructuredAgentSessionRestartResume( const continuationHost: StructuredAgentSessionContinuationHost = { ...surfaces, sessions, - stillResumable: (marker, options) => - derive([marker], 'may-be-held', options).candidates.length === 1 + conversationFence: (sessionId) => + deps.store.getRecord(sessionId) + ? structuredAgentSessionConversationFence(deps.store, sessionId) + : null, + stillResumable: (marker) => derive([marker], 'may-be-held').candidates.length === 1 } + /** One explicit action: reserve the offers, then continue each through `continueOne`, a few at + * a time. The runner counts a chat as done once `continueOne` returns. */ const run = async ( sessionIds: readonly string[] | undefined, owner: string, - afterAcquire?: (marker: AgentSessionResumeMarker) => Promise, - settlement: Omit[2], 'candidates' | 'attempts'> = { - failureAfterResume: () => null, - failureReason: () => 'agent_session_resume_refused' - } - ): Promise => { + continueOne: (marker: AgentSessionResumeMarker, continuationId: string) => Promise + ) => { // An explicit action supersedes teardown witnesses captured by this host. The durable mutation // lane below also drains a publication already in flight before completion. witnesses.clear() @@ -175,25 +160,26 @@ export function createStructuredAgentSessionRestartResume( retireSuperseded(derived.superseded) const eligible = derived.candidates.filter((candidate) => requested.has(candidate.sessionId)) if (eligible.length === 0) { - return [] + return null } const operationId = randomUUID() + const actionAt = surfaces.now() + // Tagged with its offer, so a rejected one is told apart for as long as the offer lasts. + const continuationFor = (marker: AgentSessionResumeMarker) => + restartContinuationId(marker, operationId, actionAt) const reserved = (await enqueueRecoveryOperation( () => deps.recoveryCapsule?.beginResume( eligible.map((candidate) => candidate.sessionId), operationId, - surfaces.now() + actionAt ) ?? Promise.resolve([]) )) ?? [] const markersBySession = new Map(reserved.map((marker) => [marker.sessionId, marker])) const candidates = derive(reserved, 'may-be-held').candidates - const attempts = failures.attempts(markersBySession) - - let outcomes: StructuredAgentSessionResumeOutcome[] try { - outcomes = await resumeStructuredAgentSessionsFromRestart( + const outcomes = await resumeStructuredAgentSessionsFromRestart( { admission, consumeMarker: async (sessionId) => { @@ -201,26 +187,16 @@ export function createStructuredAgentSessionRestartResume( return marker !== undefined && derive([marker], 'may-be-held').candidates.length === 1 }, resume: async (sessionId) => { - const holder = `restart-resume:${sessionId}` - try { - await surfaces.hold(sessionId, holder).catch(async (error: unknown) => { - // The reattach failure is filed like any other, so the chat must say so too. - await noteRestartReattachFailed(continuationHost, sessionId) - throw error - }) - const marker = markersBySession.get(sessionId) - if (marker) { - await afterAcquire?.(marker) - } - } finally { - attempts.observe(sessionId) - surfaces.release(sessionId, holder) + const marker = markersBySession.get(sessionId) + if (marker) { + await continueOne(marker, continuationFor(marker)) } } }, candidates, owner ) + return { operationId, outcomes, candidates, markers: markersBySession } } catch (error) { if (deps.recoveryCapsule) { await enqueueRecoveryOperation(() => @@ -231,8 +207,6 @@ export function createStructuredAgentSessionRestartResume( } throw error } - await failures.settle(operationId, outcomes, { candidates, attempts, ...settlement }) - return outcomes } const continueAfterRestart = async ( @@ -245,34 +219,46 @@ export function createStructuredAgentSessionRestartResume( failed?: StructuredAgentSessionResumeFailure[] }> => { const continued: StructuredAgentSessionContinuationOutcome[] = [] - const continuationFor = (sessionId: string) => - continued.find((outcome) => outcome.sessionId === sessionId) - const resumed = await run( - sessionIds, - owner, - async (marker) => { - continued.push( - await continueStructuredAgentSessionAfterRestart( - restartContinuationDeps(continuationHost, marker), - marker.sessionId, - marker - ) - ) - }, - { + const verdicts: Promise[] = [] + // A chat holds its slot until its agent took the continuation or its start failed, so a batch + // never starts more agents at once than the runner allows; the provider's answer comes after. + const action = await run(sessionIds, owner, async (marker, continuationId) => { + const started = await startStructuredAgentSessionContinuation( + restartContinuationDeps(continuationHost, marker), + marker.sessionId, + marker, + continuationId + ) + if ('done' in started) { + continued.push(started.done) + if (started.done.outcome === 'refused') { + throw new Error(started.done.reason ?? 'agent_session_continuation_refused') + } + return + } + verdicts.push(started.verdict().then((outcome) => void continued.push(outcome))) + }) + await Promise.all(verdicts) + const resumed = action?.outcomes ?? [] + if (action) { + await failures.settle(action.operationId, resumed, { + candidates: action.candidates, + markers: action.markers, failureAfterResume: (sessionId) => { - const outcome = continuationFor(sessionId) - // Reattached but never asked to continue: nothing confirms the agent carried on. - return outcome ? continuationFailureOutcome(outcome.outcome) : 'unconfirmed' + const outcome = continued.find((entry) => entry.sessionId === sessionId) + return outcome ? continuationFailureOutcome(outcome.outcome) : null }, failureReason: (sessionId) => { - const outcome = continuationFor(sessionId) + const outcome = continued.find((entry) => entry.sessionId === sessionId) return outcome?.reason ?? outcome?.outcome ?? 'agent_session_continuation_unknown' } - } - ) + }) + } for (const outcome of resumed) { - if (outcome.outcome !== 'resumed') { + if ( + outcome.outcome !== 'resumed' && + !continued.some((entry) => entry.sessionId === outcome.sessionId) + ) { continued.push({ sessionId: outcome.sessionId, outcome: 'refused', @@ -306,7 +292,7 @@ export function createStructuredAgentSessionRestartResume( // Do not let a teardown witness already captured in this host republish after explicit // dismissal. A later capture is a new interruption and may create a fresh offer normally. dismiss: (sessionIds) => failures.dismiss(sessionIds, witnesses.clear), - resume: (sessionIds, owner) => run(sessionIds, owner), - continueAfterRestart + continueAfterRestart, + onAgentStarted: withdrawal.onAgentStarted } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-runner.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-runner.ts index 33a07e80335c..05b3ef964508 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-runner.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-runner.ts @@ -1,13 +1,12 @@ // Delivering the explicit restart action: the one path that turns a resumable candidate back into -// a live agent. +// a working agent. // // The manual "Resume" button and the automatic setting both land here, so the two can never drift // into different eligibility or different double-fire protection. // -// Resume itself acquires a provider child, not a new send. The first resume-capable hold on a -// childless session re-acquires the provider's own conversation — Claude's `resume` by session id, -// Codex's thread id — which is native continuation. Nothing re-sends the user's prompt: that is -// what makes an agent redo work it already finished. +// A resume is the continuation send. Its delivery starts the provider on its own conversation — +// Claude's `resume` by session id, Codex's thread id — which is native continuation. Nothing +// re-sends the user's prompt: that is what makes an agent redo work it already finished. import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' import type { StructuredAgentSessionResumeCandidate } from './structured-agent-session-restart-resume-set' @@ -52,7 +51,7 @@ function resumeAdmissionOwner(error: unknown): string | null { * One resume per session at a time, whoever is asking. * * Two surfaces can reach for the same chat at once — the banner's "Resume all" and a user clicking - * one row — and both would otherwise take a hold, race the acquisition, and leave the loser's + * one row — and both would otherwise send the continuation twice, and leave the loser's * refusal looking like a real failure. The second caller is told who holds it instead. */ export class StructuredAgentSessionResumeAdmission { @@ -80,7 +79,7 @@ export type StructuredAgentSessionResumeRunnerDeps = { admission: StructuredAgentSessionResumeAdmission /** Validates this action's durable reservation. False means the candidate is no longer eligible. */ consumeMarker: (sessionId: string) => Promise - /** Acquires the provider child for the reserved session. */ + /** Continues the reserved session; resolves once its agent took the message or refused it. */ resume: (sessionId: string) => Promise concurrency?: number } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-set.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-set.ts index 245fbcf4bbf1..d5d6bd0d7ab7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-set.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-set.ts @@ -7,11 +7,10 @@ // restored thread), and every reading of those rewrites as "the work is done" dropped chats that // were owed a resume. // -// An offer ends only by the user's own actions. The ones this predicate can see — a newer message -// of theirs in that chat, or the conversation forked — are reported back as `superseded` so the -// caller DELETES the record rather than filtering it forever. What remains are structural checks -// that are not about work at all: the record still exists and this build supports it, and the -// lease is free. +// An offer ends when the chat moves on after the restart — another message accepted, or its agent +// started — or when the conversation forked. Both are reported back as `superseded` so the caller +// DELETES the record rather than filtering it forever. What remains are structural checks that are not about work at all: the record still +// exists and this build supports it, and the lease is free. import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { @@ -65,8 +64,8 @@ export type StructuredAgentSessionResumeFailure = StructuredAgentSessionResumeCa export type StructuredAgentSessionResumableSet = { candidates: StructuredAgentSessionResumeCandidate[] - /** Markers the chat has provably moved past — a newer user message, or a forked conversation. - * Every ending deletes: the caller retires these rather than re-filtering them forever. */ + /** Markers the chat has provably moved past, or whose conversation forked. Every ending deletes: + * the caller retires these rather than re-filtering them forever. */ superseded: AgentSessionResumeMarker[] } @@ -75,8 +74,8 @@ export type StructuredAgentSessionResumeSetInput = { getRecord: (sessionId: string) => AgentSessionRecord | null supportsRecord: (record: AgentSessionRecord) => boolean latestPrompt: (sessionId: string) => string - /** Undefined when the chat's journal is not readable here, which decides nothing. */ - latestUserItemId: (sessionId: string) => string | null | undefined + /** Whether the chat moved on since the offer was taken; false when its journal is not open here. */ + movedOn: (marker: AgentSessionResumeMarker) => boolean /** * Whether the lease must be free. * @@ -103,7 +102,7 @@ export function structuredAgentSessionResumableSet( continue } // A conversation that FORKED since teardown is not the one we marked, and can never be again: - // deleted like a newer message, so it cannot sit unseen forever. Compared by identity root, + // deleted, so it cannot sit unseen forever. Compared by identity root, // because a resume legitimately advances Claude's leaf and that is not a fork. const head = agentSessionProviderHandleChainHead(record.providerHandleChain) if (!head) { @@ -113,10 +112,7 @@ export function structuredAgentSessionResumableSet( superseded.push(marker) continue } - // The user moving on is the one thing that withdraws the offer. Anything the provider does on - // its own after reattaching — a turn it opens, a prompt, restated rows — is not. - const latestUserItemId = input.latestUserItemId(marker.sessionId) - if (latestUserItemId !== undefined && latestUserItemId !== marker.latestUserItemId) { + if (input.movedOn(marker)) { superseded.push(marker) continue } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts index e5b65009634a..e359e31157b9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts @@ -145,7 +145,7 @@ export type HarnessJournal = { appendItem: (envelope: unknown, body: { kind: string; text: string }) => Promise } -export type HarnessSession = { journal: HarnessJournal; hasProviderChild: boolean; fence?: number } +export type HarnessSession = { journal: HarnessJournal; child: { fence: number } | null } export function journal( items: AgentJournalRenderItem[], @@ -203,13 +203,12 @@ export function resumableSet(input: { markers: AgentSessionResumeMarker[] items?: AgentJournalRenderItem[] chain?: AgentSessionRecord['providerHandleChain'] - latestUserItemId?: string | null }) { return structuredAgentSessionResumableSet({ markers: input.markers, getRecord: () => record(input.chain === undefined ? {} : { chain: input.chain }), supportsRecord: () => true, latestPrompt: () => 'fix the auth bug', - latestUserItemId: () => input.latestUserItemId ?? null + movedOn: () => false }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts index 07ec6b81599f..4004502e4081 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts @@ -11,7 +11,27 @@ import type { AgentSessionMutationResult, AgentSessionSendResult } from '../../../shared/agent-session-wire' -import type { StructuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-host' +import { MAX_TIMER_DELAY_MS } from '../../../shared/timer-delay' +import type { SendSettlementWaitOptions } from './structured-agent-session-send-settlement' + +export type StructuredAgentSessionRestartResumeSurfaces = { + revealSession: (sessionId: string) => Promise<{ readable: boolean }> + send: (input: { + envelope: AgentSessionMutationEnvelope + body: AgentJournalMessageItem + beforeRun?: () => void + }) => Promise> + awaitSendSettlement: ( + sessionId: string, + clientMessageId: string + ) => Promise<{ value: AgentSessionSendResult } | undefined> + awaitSendHandedOver: ( + sessionId: string, + clientMessageId: string + ) => Promise<{ value: AgentSessionSendResult } | undefined> + onNoteFailed: (sessionId: string, error: unknown) => void + now: () => number +} /** The caller key the continuation sends under, so its writes are attributable to Orca itself. */ export const STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER = @@ -21,8 +41,6 @@ export const STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER = * dependency, and nothing outside this list is reachable from here. */ type RestartResumeHostBindings = { revealSession: (sessionId: string) => Promise<{ readable: boolean }> - hold: (sessionId: string, holderId: string) => Promise - release: (sessionId: string, holderId: string) => void send: ( caller: { callerKey: string }, params: { @@ -34,7 +52,8 @@ type RestartResumeHostBindings = { /** The host's existing settlement waiter; a send returns while its dispatch is still pending. */ waitForSendSettlement: ( sessionId: string, - clientMessageId: string + clientMessageId: string, + options: SendSettlementWaitOptions ) => Promise<{ value: AgentSessionSendResult } | undefined> } @@ -44,11 +63,19 @@ export function structuredAgentSessionRestartResumeSurfaces( ): StructuredAgentSessionRestartResumeSurfaces { return { revealSession: host.revealSession, - hold: host.hold, - release: host.release, send: (params) => host.send({ callerKey: STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER }, params), - awaitSendSettlement: host.waitForSendSettlement, + // Accepted like any send, so its verdict is its delivery, however long the start takes; the + // wait ends when the submission settles or the session closes. + awaitSendSettlement: (sessionId, clientMessageId) => + host.waitForSendSettlement(sessionId, clientMessageId, { budgetMs: MAX_TIMER_DELAY_MS }), + // How long a restart batch holds a chat's slot: until the agent took the message or its start + // failed. Undefined — the session closed, or too many waited — frees the slot too. + awaitSendHandedOver: (sessionId, clientMessageId) => + host.waitForSendSettlement(sessionId, clientMessageId, { + until: 'handed-over', + budgetMs: MAX_TIMER_DELAY_MS + }), onNoteFailed: () => console.warn('[structured-agent-session] restart continuation attribution failed'), now diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts index 1d6a8a1fc5ec..efcd70345a83 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts @@ -17,6 +17,7 @@ import { structuredAgentSessionWorkingAtStop } from './structured-agent-session- import { CLAUDE_ROOT, claudeRecord, + EPOCH, HANDLE_ROOT, journal, TEARDOWN_CURRENT, @@ -48,7 +49,7 @@ describe('deriving what was working at teardown', () => { it('marks a session this host was running a turn for', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -64,6 +65,7 @@ describe('deriving what was working at teardown', () => { recordedAt: NOW, trigger: 'quit', teardownId: TEARDOWN_CURRENT, + journalCursor: { epoch: EPOCH, sequence: 1 }, providerHandleRoot: HANDLE_ROOT, latestUserItemId: null, activity: { state: 'working', prompts: [], tasks: [] } @@ -74,7 +76,7 @@ describe('deriving what was working at teardown', () => { it('carries the update trigger so the surface can say the restart was not the user choice', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -89,7 +91,7 @@ describe('deriving what was working at teardown', () => { it('marks nothing for an idle session', () => { expect( markersAtTeardown({ - sessions: new Map([[SESSION, { journal: journal([]), hasProviderChild: true }]]), + sessions: new Map([[SESSION, { journal: journal([]), child: { fence: 1 } }]]), getRecord: () => record(), backgroundTasks: () => undefined, trigger: 'quit', @@ -103,7 +105,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -115,12 +117,12 @@ describe('deriving what was working at teardown', () => { }) // The user's stated fear. A journal restored for READING carries whatever `running` row an older - // crash left behind, and it is the live `hasProviderChild` — not that row — that decides. + // crash left behind, and it is the live `child` — not that row — that decides. it('marks nothing for a stale running row this host was not executing', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: false }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: null }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -141,7 +143,7 @@ describe('deriving what was working at teardown', () => { SESSION, { journal: journal([turnItem('turn-1', 'running'), pendingApproval()]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -165,7 +167,7 @@ describe('deriving what was working at teardown', () => { it('marks a settled lead whose subagent was still running, anchored on its last turn', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -188,7 +190,7 @@ describe('deriving what was working at teardown', () => { it('records only the live rows of the roster, bounded', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -213,7 +215,7 @@ describe('deriving what was working at teardown', () => { it('marks a settled lead whose only live work is a monitor', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [{ id: 'task-m', kind: 'monitor', name: 'ci-watch' }], @@ -230,7 +232,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -248,7 +250,7 @@ describe('deriving what was working at teardown', () => { it('records the identity root so an advancing Claude leaf cannot invalidate the marker', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => claudeRecord(null), backgroundTasks: () => undefined, @@ -264,7 +266,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record({ chain: [] }), backgroundTasks: () => undefined, @@ -285,7 +287,7 @@ describe('deriving what was working at teardown', () => { SESSION, { journal: journal([], false, [submission('msg-1', 'pending')]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -310,7 +312,7 @@ describe('deriving what was working at teardown', () => { journal: journal([turnItem('turn-0', 'completed')], false, [ submission('msg-1', 'pending') ]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -324,6 +326,67 @@ describe('deriving what was working at teardown', () => { expect(recorded?.work).toEqual({ kind: 'submission', id: 'msg-1' }) }) + // Accepted while the agent was starting and never handed over: the chat shows working, but no + // agent had the message, and quit rejects it as never sent. + it('marks nothing for a session whose only work is a message still queued', () => { + const queued = { ...submission('msg-1', 'pending'), handoverRecorded: true as const } + expect( + markersAtTeardown({ + sessions: new Map([ + [ + SESSION, + { + journal: journal([turnItem('turn-0', 'completed')], false, [queued]), + child: { fence: 1 } + } + ] + ]), + getRecord: () => claudeRecord(null), + backgroundTasks: () => undefined, + trigger: 'quit', + teardownId: TEARDOWN_CURRENT, + now: NOW + }) + ).toEqual([]) + }) + + it('marks a handed-over message over a newer queued one, and a turn a queued one waits behind', () => { + const handedOver = { + ...submission('msg-1', 'pending'), + handoverRecorded: true as const, + handedOverAt: NOW + } + const queued = { ...submission('msg-2', 'pending'), handoverRecorded: true as const } + const markers = markersAtTeardown({ + sessions: new Map([ + [ + SESSION, + { + journal: journal([turnItem('turn-0', 'completed')], false, [handedOver, queued]), + child: { fence: 1 } + } + ], + [ + 'session-running', + { + journal: journal([turnItem('turn-1', 'running')], false, [queued]), + child: { fence: 1 } + } + ] + ]), + getRecord: () => claudeRecord(null), + backgroundTasks: () => undefined, + trigger: 'quit', + teardownId: TEARDOWN_CURRENT, + now: NOW + }) + + expect(markers.map((entry) => entry.work)).toEqual([ + { kind: 'submission', id: 'msg-1' }, + { kind: 'turn', id: 'turn-1' } + ]) + }) + // Once a turn exists it is the better identity: it is what eviction rewrites, so it is what the // journal can be asked about at launch. it('prefers the running turn over the send that opened it', () => { @@ -335,7 +398,7 @@ describe('deriving what was working at teardown', () => { journal: journal([turnItem('turn-1', 'running')], false, [ submission('msg-1', 'accepted') ]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -390,7 +453,7 @@ describe('the resumable set', () => { getRecord: () => claudeRecord('5aed93d6-advanced-leaf'), supportsRecord: () => true, latestPrompt: () => '', - latestUserItemId: () => null + movedOn: () => false }) expect(candidates).toHaveLength(1) @@ -405,14 +468,14 @@ describe('the resumable set', () => { getRecord: () => claudeRecord(null, 'prov-session-2'), supportsRecord: () => true, latestPrompt: () => '', - latestUserItemId: () => null + movedOn: () => false }) expect(set.candidates).toEqual([]) expect(set.superseded).toEqual([forked]) }) - // An offer has no expiry: it ends only by the user's own actions, however old it is. + // An offer has no expiry, however old it is. it('still offers a months-old marker', () => { const monthsAgo = NOW - 90 * 24 * 60 * 60 * 1000 expect(resumableSet({ markers: [marker({ recordedAt: monthsAgo })] }).candidates).toHaveLength( @@ -420,16 +483,6 @@ describe('the resumable set', () => { ) }) - // The user moving on is what withdraws an offer — and it is reported for DELETION, not merely - // filtered, so the record does not have to be re-filtered on every read forever. - it('withdraws and reports for deletion once the user has sent a newer message', () => { - const withdrawn = marker() - const set = resumableSet({ markers: [withdrawn], latestUserItemId: 'user-newer' }) - - expect(set.candidates).toEqual([]) - expect(set.superseded).toEqual([withdrawn]) - }) - // Structural refusals are NOT endings: a record this host cannot see right now must not delete // a durable offer the user still owns. it('does not report a structurally refused marker for deletion', () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts index 884fba78a4ec..ef9ef80687d6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts @@ -94,6 +94,8 @@ async function restartWithPersistedTurn(): Promise { const store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) const host = createHost(store) expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + // A restart owes a row to every listed chat; an unlisted one waits for its first read. + await store.setSessionTabVisibility(SESSION, true) const body = hostTestMessage('persisted conversation') await host.send(CALLER, { envelope: sendEnvelope(store, { body }), body }) await host.flushAllStreamedEvents() @@ -112,7 +114,7 @@ describe('structured session restart status publication', () => { it('projects the persisted turn of a session restored without a provider', async () => { const restarted = await restartWithPersistedTurn() - await restarted.restoreReadableSessions() + await restarted.restoreStartupSessions() const events: AgentSessionStatusEvent[] = [] restarted.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) }) @@ -138,7 +140,7 @@ describe('structured session restart status publication', () => { restarted.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) }) expect(events).toEqual([{ type: 'snapshot', sessions: [] }]) - await restarted.restoreReadableSessions() + await restarted.restoreStartupSessions() // The restore wiring publishes; without it this list never hears about the session at all. expect(events.at(-1)).toEqual({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-teardown.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-teardown.test.ts index cc40d4831875..bf740ac405b2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-teardown.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-teardown.test.ts @@ -30,7 +30,7 @@ it.each(['beginTeardown', 'captureBeforeStop', 'recordMarkers'] as const)( ) expect(warning.mock.calls.flat().map(String).join(' ')).not.toContain(failure.message) expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - expect(() => host.journalSnapshot(SESSION)).toThrow('agent_session_ownership_unknown') + await expect(host.journalSnapshot(SESSION)).rejects.toThrow('agent_session_ownership_unknown') } finally { operation.mockRestore() warning.mockRestore() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts index 1b0540183710..5f799e4498ea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts @@ -10,7 +10,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { randomUUID } from 'node:crypto' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { attachParamsForRecord } from './structured-agent-session-read-restore' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' export function isResumableStructuredAgentSessionRecord(record: AgentSessionRecord): boolean { return ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts index 291a6e0c53eb..49a116ec7f7a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts @@ -3,8 +3,8 @@ * * The reveal path is the only way an Agent Session History row reaches a chat whose tab this * process never published — a chat closed cleanly, or one this process has not opened since - * launch. It is exercised here through the readable restorer rather than the whole host, because - * what it must get right is which records it accepts and what it does when the journal is gone. + * launch. It opens the conversation through the host's accessor and starts no agent; what it must + * get right is which records it accepts and what it answers when the journal cannot be opened. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -13,8 +13,6 @@ import { agentSessionLeaseFixture, agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' -import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' -import * as readRestore from './structured-agent-session-read-restore' import * as providerSupport from './structured-agent-session-provider-support' import { revealStructuredAgentSession } from './structured-agent-session-reveal' @@ -35,114 +33,10 @@ function recordFor(provider: 'claude' | 'codex', sessionId: string): AgentSessio } } -function harness( - records: AgentSessionRecord[], - options: { supports?: (record: AgentSessionRecord) => boolean } = {} -) { - const live = new Map() - const serializedIds: string[] = [] - const serialize = (sessionId: string, task: () => Promise): Promise => { - serializedIds.push(sessionId) - return task() - } - const restorer = new StructuredAgentSessionReadableRestorer({ - store: { - getRecord: (sessionId: string) => records.find((r) => r.sessionId === sessionId) ?? null, - listRecords: () => records - } as never, - journalRoot: '/journals', - supportsRecord: options.supports ?? (() => true), - reconcile: async () => null, - resolveRecovery: async () => undefined, - serialize, - hasSession: (sessionId) => live.has(sessionId), - onReadable: (sessionId, restored) => live.set(sessionId, restored), - retrySettlement: async () => true - }) - return { restorer, live, serializedIds } -} - -const readable = { journal: {}, params: {}, fence: 1 } as never - afterEach(() => { vi.restoreAllMocks() }) -describe('revealing one structured session on demand', () => { - beforeEach(() => { - vi.spyOn(readRestore, 'restoreStructuredAgentSessionRead').mockResolvedValue(readable) - }) - - it.each(['claude', 'codex'] as const)('restores a persisted %s chat', async (provider) => { - const sessionId = `session-${provider}` - const { restorer, live } = harness([recordFor(provider, sessionId)]) - - await expect(restorer.restoreOne(sessionId)).resolves.toBe(true) - expect(live.has(sessionId)).toBe(true) - }) - - it('does not need the startup sweep to have run, or run it', async () => { - // The whole point: `restore()` is latched to once per process, and a surface asking later must - // not be answered from that latch — nor trip it, which would skip every other record. - const records = [recordFor('codex', 'session-asked'), recordFor('claude', 'session-untouched')] - const { restorer, live } = harness(records) - - await restorer.restoreOne('session-asked') - - expect(live.has('session-asked')).toBe(true) - expect(live.has('session-untouched')).toBe(false) - }) - - it('serializes against the session it restores', async () => { - const { restorer, serializedIds } = harness([recordFor('claude', 'session-serialized')]) - - await restorer.restoreOne('session-serialized') - - // Ordering against close/eviction is the task queue's job, so the restore must be inside it. - expect(serializedIds).toEqual(['session-serialized']) - }) - - it('returns the live session instead of restoring over it', async () => { - const { restorer, live } = harness([recordFor('codex', 'session-live')]) - live.set('session-live', readable) - - await expect(restorer.restoreOne('session-live')).resolves.toBe(true) - expect(readRestore.restoreStructuredAgentSessionRead).not.toHaveBeenCalled() - }) - - it('refuses a record no adapter supports', async () => { - const { restorer } = harness([recordFor('codex', 'session-unsupported')], { - supports: () => false - }) - - await expect(restorer.restoreOne('session-unsupported')).resolves.toBe(false) - expect(readRestore.restoreStructuredAgentSessionRead).not.toHaveBeenCalled() - }) - - it('refuses a session this host holds no record for', async () => { - const { restorer } = harness([]) - - await expect(restorer.restoreOne('session-absent')).resolves.toBe(false) - }) -}) - -describe('a record whose journal cannot be read', () => { - it('reports not-readable without throwing, for either provider', async () => { - // A chat whose journal predates the SQLite store restores to nothing here. That is not a - // refusal: attach still recovers it, so the caller publishes the tab and lets the pane's hold - // finish the job. Throwing, or reporting success, would both be wrong. - vi.spyOn(readRestore, 'restoreStructuredAgentSessionRead').mockResolvedValue(null) - const { restorer, live } = harness([ - recordFor('claude', 'session-no-journal-claude'), - recordFor('codex', 'session-no-journal-codex') - ]) - - await expect(restorer.restoreOne('session-no-journal-claude')).resolves.toBe(false) - await expect(restorer.restoreOne('session-no-journal-codex')).resolves.toBe(false) - expect(live.size).toBe(0) - }) -}) - describe('the host answer a client acts on', () => { beforeEach(() => { // Eligibility is the router's call; these cases are about what the answer carries. @@ -154,39 +48,45 @@ describe('the host answer a client acts on', () => { return { ...base, location: { ...base.location, workspaceId } } } - it("answers with the record's own workspace and provider, never a caller's", async () => { - // The security property: a client sends only a session id, so the tab cannot be aimed at - // another workspace by asking for one. - const stored = record('claude', 'workspace-from-record') - - await expect( - revealStructuredAgentSession( - { store: { getRecord: () => stored } as never, adapter: {} as never }, - 'session-answered', - () => true, - async () => true - ) - ).resolves.toEqual({ - sessionId: 'session-answered', - workspaceId: 'workspace-from-record', - agent: 'claude', - readable: true - }) - }) - - it('refuses a session this host holds no record for', async () => { + it.each(['claude', 'codex'] as const)( + "answers a %s chat with the record's own workspace and provider, never a caller's", + async (provider) => { + // The security property: a client sends only a session id, so the tab cannot be aimed at + // another workspace by asking for one. + const stored = record(provider, 'workspace-from-record') + const open = vi.fn(async () => undefined) + + await expect( + revealStructuredAgentSession( + { store: { getRecord: () => stored } as never, adapter: {} as never }, + 'session-answered', + open + ) + ).resolves.toEqual({ + sessionId: 'session-answered', + workspaceId: 'workspace-from-record', + agent: provider, + readable: true + }) + expect(open).toHaveBeenCalledExactlyOnceWith('session-answered') + } + ) + + it('refuses a session this host holds no record for, opening nothing', async () => { + const open = vi.fn(async () => undefined) await expect( revealStructuredAgentSession( { store: { getRecord: () => null } as never, adapter: {} as never }, 'session-absent', - () => false, - async () => false + open ) ).rejects.toThrow('agent_session_identity_required') + expect(open).not.toHaveBeenCalled() }) it('refuses a record no adapter of this host supports', async () => { vi.mocked(providerSupport.adapterSupportsRecord).mockReturnValue(false) + const open = vi.fn(async () => undefined) await expect( revealStructuredAgentSession( @@ -195,14 +95,14 @@ describe('the host answer a client acts on', () => { adapter: {} as never }, 'session-answered', - () => false, - async () => false + open ) ).rejects.toThrow('structured_agent_session_unsupported') + expect(open).not.toHaveBeenCalled() }) - it('answers not-readable without refusing when the journal could not be restored', async () => { - // The tab is still worth publishing: attach recovers what read restore cannot. + it('answers not-readable without refusing when the journal could not be opened', async () => { + // The tab is still worth publishing: the chat shows the failure and keeps retrying the read. await expect( revealStructuredAgentSession( { @@ -210,26 +110,10 @@ describe('the host answer a client acts on', () => { adapter: {} as never }, 'session-answered', - () => false, - async () => false + async () => { + throw new Error('journal unreadable') + } ) ).resolves.toMatchObject({ readable: false, agent: 'codex' }) }) - - it('does not restore over a session that is already live', async () => { - const restoreReadable = vi.fn(async () => true) - - await expect( - revealStructuredAgentSession( - { - store: { getRecord: () => record('codex', 'workspace-1') } as never, - adapter: {} as never - }, - 'session-answered', - () => true, - restoreReadable - ) - ).resolves.toMatchObject({ readable: true }) - expect(restoreReadable).not.toHaveBeenCalled() - }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index bc36df76df8a..021325dd0cea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -6,27 +6,21 @@ // launch — is reachable in Agent Session History by id and by nothing else. This is the lookup that // turns that id back into something a client can publish. // -// It is deliberately the whole of what reveal does on the host. It takes no hold: a provider child -// exists because a surface asked for one, and the chat pane asks when it binds. And a journal it -// cannot read is not a refusal — a chat whose journal predates the SQLite store restores to nothing -// here, yet attach still recovers it, so the tab is worth publishing either way. +// It is deliberately the whole of what reveal does on the host. It starts no provider child: only +// a send does. And a journal it cannot open is not a refusal — the chat shows that failure with a +// Retry, so the tab is worth publishing either way. import { adapterSupportsRecord } from './structured-agent-session-provider-support' -import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' -import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' import type { StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession, StructuredAgentSessionReveal } from './structured-agent-session-host-types' -import { retryPendingStructuredAgentSessionSettlement } from './structured-agent-session-settlement-retry' -/** Throws its refusal as the code itself, matching `resumeHeldStructuredAgentSession`. */ +/** Throws its refusal as the code itself. */ export async function revealStructuredAgentSession( deps: Pick, sessionId: string, - hasSession: (sessionId: string) => boolean, - restoreReadable: (sessionId: string) => Promise + openConversation: (sessionId: string) => Promise ): Promise { const record = deps.store.getRecord(sessionId) if (!record) { @@ -36,9 +30,12 @@ export async function revealStructuredAgentSession( throw new Error('structured_agent_session_unsupported') } // Lease state is not consulted on purpose: this neither claims the lease nor spawns a child, so a - // contested or reconciling chat still reveals and the hold that follows adjudicates it. Refusing + // contested or reconciling chat still reveals and the send that follows adjudicates it. Refusing // here would hide the one view of a session a user needs when its ownership is in doubt. - const readable = hasSession(sessionId) || (await restoreReadable(sessionId)) + const readable = await openConversation(sessionId).then( + () => true, + () => false + ) return { sessionId, // From the record, never from a caller: a client that knows only a session id must not be able @@ -48,42 +45,3 @@ export async function revealStructuredAgentSession( readable } } - -/** - * The host's whole readable-restore surface: the startup sweep and the on-demand reveal. - * - * Bundled the way the lifetime collaborators are, because the two share the restorer - * and differ only in who is asking — startup, once, for everything; a surface, later, for one. - */ -export function createStructuredAgentSessionHostRestore( - deps: StructuredAgentSessionHostDeps, - sessions: Map, - now: () => number, - wiring: Omit< - ConstructorParameters[0], - 'store' | 'journalRoot' | 'supportsRecord' | 'retrySettlement' - > -): { - restoreReadableSessions: (sessionIds?: readonly string[]) => Promise - revealSession: (sessionId: string) => Promise - /** One session, for a caller already inside its serialize. */ - restoreReadableUnderSerialize: (sessionId: string) => Promise -} { - const restorer = new StructuredAgentSessionReadableRestorer({ - store: deps.store, - journalRoot: deps.journalRoot, - supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), - retrySettlement: (sessionId, params) => - retryPendingStructuredAgentSessionSettlement({ deps, sessions, sessionId, params, now }), - ...wiring - }) - const gate = new StructuredAgentSessionRestartRestoreGate() - return { - restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), - revealSession: (sessionId) => - revealStructuredAgentSession(deps, sessionId, wiring.hasSession, (id) => - restorer.restoreOne(id) - ), - restoreReadableUnderSerialize: (sessionId) => restorer.restoreOneUnderSerialize(sessionId) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts new file mode 100644 index 000000000000..a596cc5c9edb --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts @@ -0,0 +1,225 @@ +// A Codex rewind whose outcome only the provider can prove, found on a chat at rest: nothing on +// screen will start the agent that settles it, so the next send does, and its message is kept. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { + AgentSessionDispatchOutcome, + StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import { + HOST_TEST_NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const caller = { callerKey: 'desktop' } +const KEPT = { provider: 'codex' as const, threadId: THREAD, turnId: 'kept', ordinal: 0 } + +let directory: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let sink: StructuredAgentSessionEventSink +let acquires = 0 +const rewind = vi.fn>() +const recoverRewind = vi.fn>() +const rewindSupport = vi.fn>() +const dispatch = vi.fn() + +function adapter(): StructuredAgentSessionAdapter { + return { + supportsCreate: (_location, agent) => agent === 'codex', + supportsLocation: () => true, + acquire: async (input) => { + acquires += 1 + sink = input.events! + return { + process: { + hostId: 'local', + pid: 4000 + acquires, + processStartTimeMs: HOST_TEST_NOW, + spawnToken: input.spawnToken + }, + acquisitionGeneration: `generation-${acquires}`, + link: { + linkId: `link-${acquires}`, + mintedAtFence: input.fence, + observedAt: HOST_TEST_NOW, + origin: acquires === 1 ? 'created' : 'resumed', + handle: { provider: 'codex', threadId: THREAD } + } + } + }, + dispatch, + cancelTurn: async () => ({ cancelled: false }), + answerPrompt: async () => {}, + setOption: async () => {}, + rewindSupport, + rewind, + recoverRewind, + releaseAcquisition: async () => true, + closeSession: async () => true + } +} + +function openHost(): StructuredAgentSessionHost { + return new StructuredAgentSessionHost({ + store, + adapter: adapter(), + journalRoot: directory, + claimKeyId: 'key', + now: () => HOST_TEST_NOW, + probeOwner: async () => ({ outcome: 'exit-observed' }), + idleSweep: { intervalMs: 3_600_000 } + }) +} + +beforeEach(async () => { + resetHostTestOperationIds() + acquires = 0 + rewind.mockReset() + recoverRewind.mockReset().mockResolvedValue({ + ok: true, + items: [{ identity: KEPT, body: hostTestMessage('verified history') }] + }) + rewindSupport.mockReset().mockReturnValue({ supported: true }) + dispatch.mockReset().mockImplementation(async (input): Promise => ({ + state: 'accepted', + providerIdentity: { + provider: 'codex', + threadId: THREAD, + turnId: input.clientMessageId, + ordinal: 1 + } + })) + directory = await mkdtemp(join(tmpdir(), 'orca-rewind-rest-')) + store = await AgentSessionRecordStore.open({ + directory: join(directory, 'store'), + hostId: 'local' + }) + host = openHost() +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(directory, { recursive: true, force: true }) +}) + +function fence(): number { + return store.getRecord(SESSION)!.lease.runtimeFence +} + +function rewindParams(itemId: string, expectedEpoch: string) { + return { + itemId, + expectedEpoch, + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence(), + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.rewind', + sessionId: SESSION, + fields: { itemId, expectedEpoch } + }) + } + } +} + +function sendParams(text: string) { + const body = hostTestMessage(text) + return { + body, + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence(), + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + } + } +} + +/** A chat whose rewind the provider applied, whose Orca side never finished, reopened at rest. */ +async function interruptedRewindAtRest(): Promise { + expect(await host.attach(caller, hostTestAttachParams(null))).toMatchObject({ ok: true }) + const drop = { ...KEPT, turnId: 'drop' } + sink.appendItem(KEPT, hostTestMessage('verified history')) + sink.appendItem(drop, hostTestMessage('to be rewound')) + await host.flushStreamedEvents(SESSION) + rewind.mockImplementation(async (input) => { + await input.onReverted?.() + throw new Error('history unavailable') + }) + const epoch = (await host.journalSnapshot(SESSION)).cursor.epoch + await expect(host.rewind(caller, rewindParams(agentJournalItemKey(drop), epoch))).rejects.toThrow( + 'history unavailable' + ) + expect(store.getRecord(SESSION)?.rewind).toMatchObject({ phase: 'prepared' }) + await host.flushAllStreamedEvents() + store = await AgentSessionRecordStore.open({ + directory: join(directory, 'store'), + hostId: 'local' + }) + host = openHost() +} + +describe('an interrupted Codex rewind on a chat at rest (R16)', () => { + it('is settled by the start a send makes, and the message is delivered after it', async () => { + await interruptedRewindAtRest() + expect(host.hasSession(SESSION)).toBe(false) + const before = acquires + + const sent = await host.send(caller, sendParams('after the rewind')) + expect(sent).toMatchObject({ ok: true }) + expect(acquires - before).toBe(1) + expect(recoverRewind).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.rewind?.phase).toBe('completed') + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) + const snapshot = await host.journalSnapshot(SESSION) + const texts = snapshot.items.flatMap((item) => + item.body.kind === 'message' + ? item.body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])) + : [] + ) + // The recovered history, then the new message — nothing the rewind dropped, nothing lost. + expect(texts).toEqual(['verified history', 'after the rewind']) + expect(snapshot.submissions.at(-1)?.dispatchState).toBe('accepted') + }) +}) + +describe('a rewind asked of a chat at rest (P2-23)', () => { + it('starts the agent first and answers with what the provider says', async () => { + expect(await host.attach(caller, hostTestAttachParams(null))).toMatchObject({ ok: true }) + sink.appendItem(KEPT, hostTestMessage('kept')) + await host.flushStreamedEvents(SESSION) + const epoch = (await host.journalSnapshot(SESSION)).cursor.epoch + await host.flushAllStreamedEvents() + store = await AgentSessionRecordStore.open({ + directory: join(directory, 'store'), + hostId: 'local' + }) + host = openHost() + const before = acquires + rewindSupport.mockReturnValue({ supported: false, reason: 'history-not-paginated' }) + + const result = await host.rewind(caller, rewindParams(agentJournalItemKey(KEPT), epoch)) + expect(acquires - before).toBe(1) + expect(result).toMatchObject({ ok: false, refusal: { rewindReason: 'history-not-paginated' } }) + expect(rewind).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts index b82248cb06ff..a8ded60e6ffb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts @@ -139,6 +139,14 @@ async function seed(acceptedSubmissions = false) { } }) ).toMatchObject({ ok: true }) + // Accepted into the conversation first; the delivery loop hands it over after. + await vi.waitFor(async () => + expect( + (await host.journalSnapshot(HOST_TEST_SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientOperationId + )?.dispatchState + ).toBe('accepted') + ) if (i === 1) { selectedItemId = agentJournalSubmissionKey(clientOperationId) } @@ -149,10 +157,8 @@ async function seed(acceptedSubmissions = false) { await host.flushStreamedEvents(HOST_TEST_SESSION) return selectedItemId } -function params( - itemId: string, - expectedEpoch = host.journalSnapshot(HOST_TEST_SESSION).cursor.epoch -) { +async function params(itemId: string, epoch?: string) { + const expectedEpoch = epoch ?? (await host.journalSnapshot(HOST_TEST_SESSION)).cursor.epoch return { itemId, expectedEpoch, @@ -173,14 +179,14 @@ describe('host rewind', () => { it('resolves accepted codex user submissions to provider targets', async () => { const target = await seed(true) expect(target.startsWith('orca:')).toBe(true) - expect(await host.rewind(caller, params(target))).toMatchObject({ ok: true }) - expect(host.journalSnapshot(HOST_TEST_SESSION).items).toHaveLength(1) + expect(await host.rewind(caller, await params(target))).toMatchObject({ ok: true }) + expect((await host.journalSnapshot(HOST_TEST_SESSION)).items).toHaveLength(1) expect(rewind).toHaveBeenCalledWith(expect.objectContaining({ beforeTurnId: 'drop' })) }) it('finishes a durable provider success on reattach without repeating the provider mutation', async () => { const target = await seed() - const request = params(target) + const request = await params(target) const replace = vi .spyOn(AgentSessionJournal.prototype, 'replaceEpochItems') .mockRejectedValueOnce(new Error('disk failed')) @@ -189,7 +195,7 @@ describe('host rewind', () => { replace.mockRestore() const fence = store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence expect(await host.attach(caller, hostTestAttachParams(fence))).toMatchObject({ ok: true }) - expect(host.journalSnapshot(HOST_TEST_SESSION).items).toHaveLength(1) + expect((await host.journalSnapshot(HOST_TEST_SESSION)).items).toHaveLength(1) expect(store.getRecord(HOST_TEST_SESSION)?.rewind?.phase).toBe('completed') expect(await host.rewind(caller, request)).toMatchObject({ ok: true, replayed: true }) expect(rewind).toHaveBeenCalledTimes(1) @@ -197,13 +203,13 @@ describe('host rewind', () => { it('retries complete hydration after native acknowledgement without committing partial history', async () => { const target = await seed() - const before = host.journalSnapshot(HOST_TEST_SESSION) + const before = await host.journalSnapshot(HOST_TEST_SESSION) rewind.mockImplementation(async (input) => { await input.onReverted?.() throw new Error('history unavailable') }) - await expect(host.rewind(caller, params(target))).rejects.toThrow('history unavailable') - expect(host.journalSnapshot(HOST_TEST_SESSION)).toEqual(before) + await expect(host.rewind(caller, await params(target))).rejects.toThrow('history unavailable') + expect(await host.journalSnapshot(HOST_TEST_SESSION)).toEqual(before) expect(store.getRecord(HOST_TEST_SESSION)?.rewind).toMatchObject({ phase: 'prepared', providerApplied: true @@ -222,8 +228,8 @@ describe('host rewind', () => { hostTestAttachParams(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) ) ).toMatchObject({ ok: true }) - expect(host.journalSnapshot(HOST_TEST_SESSION).items).toHaveLength(1) - expect(host.journalSnapshot(HOST_TEST_SESSION).items[0]?.body).toEqual( + expect((await host.journalSnapshot(HOST_TEST_SESSION)).items).toHaveLength(1) + expect((await host.journalSnapshot(HOST_TEST_SESSION)).items[0]?.body).toEqual( hostTestMessage('verified history') ) expect(recoverRewind).toHaveBeenCalledTimes(2) @@ -238,9 +244,9 @@ describe('host rewind', () => { finish = () => resolve({ ok: true }) }) ) - const first = host.rewind(caller, params(target)) - const second = host.rewind(caller, params(target)) - await vi.waitFor(() => expect(finish).toBeTypeOf('function')) + const first = host.rewind(caller, await params(target)) + const second = host.rewind(caller, await params(target)) + await vi.waitFor(async () => expect(finish).toBeTypeOf('function')) finish() expect(await first).toMatchObject({ ok: true }) expect(await second).toMatchObject({ ok: false, refusal: { rewindReason: 'stale-epoch' } }) @@ -248,11 +254,13 @@ describe('host rewind', () => { }) it('replaces the epoch with the retained prefix and replays without another provider call', async () => { const target = await seed() - const request = params(target) + const request = await params(target) const result = await host.rewind(caller, request) expect(result).toMatchObject({ ok: true }) - expect(host.journalSnapshot(HOST_TEST_SESSION).items).toHaveLength(1) - expect(host.journalSnapshot(HOST_TEST_SESSION).cursor.epoch).not.toBe(request.expectedEpoch) + expect((await host.journalSnapshot(HOST_TEST_SESSION)).items).toHaveLength(1) + expect((await host.journalSnapshot(HOST_TEST_SESSION)).cursor.epoch).not.toBe( + request.expectedEpoch + ) expect(await host.rewind(caller, request)).toMatchObject({ ok: true, replayed: true }) expect(rewind).toHaveBeenCalledTimes(1) }) @@ -262,7 +270,7 @@ describe('host rewind', () => { { provider: 'orca', clientMessageId: 'active' }, { kind: 'status', text: 'working', turnLifecycle: { turnId: 'active', state: 'running' } } ) - expect(await host.rewind(caller, params(target))).toMatchObject({ + expect(await host.rewind(caller, await params(target))).toMatchObject({ ok: false, refusal: { rewindReason: 'busy' } }) @@ -270,11 +278,11 @@ describe('host rewind', () => { }) it('refuses stale epochs and targets from another provider', async () => { const target = await seed() - expect(await host.rewind(caller, params(target, 'old-epoch'))).toMatchObject({ + expect(await host.rewind(caller, await params(target, 'old-epoch'))).toMatchObject({ ok: false, refusal: { rewindReason: 'stale-epoch' } }) - expect(await host.rewind(caller, params('claude:foreign'))).toMatchObject({ + expect(await host.rewind(caller, await params('claude:foreign'))).toMatchObject({ ok: false, refusal: { rewindReason: 'invalid-target' } }) @@ -282,18 +290,18 @@ describe('host rewind', () => { }) it('keeps a failed hydration epoch intact and blocks sends and duplicate rewind', async () => { const target = await seed() - const request = params(target) - const before = host.journalSnapshot(HOST_TEST_SESSION) + const request = await params(target) + const before = await host.journalSnapshot(HOST_TEST_SESSION) rewind.mockRejectedValue(new Error('hydration failed')) await expect(host.rewind(caller, request)).rejects.toThrow('hydration failed') - expect(host.journalSnapshot(HOST_TEST_SESSION)).toEqual(before) + expect(await host.journalSnapshot(HOST_TEST_SESSION)).toEqual(before) expect(await host.rewind(caller, request)).toMatchObject({ ok: false, refusal: { code: 'agent_session_operation_unknown' } }) const body = hostTestMessage('new prompt') const envelope = { - ...params(target).envelope, + ...(await params(target)).envelope, payloadFingerprint: computeAgentSessionPayloadFingerprint({ method: 'agentSession.send', sessionId: HOST_TEST_SESSION, @@ -318,9 +326,9 @@ describe('host rewind', () => { it('clears an unapplied prepared rewind after observing the target still present', async () => { const target = await seed() - const before = host.journalSnapshot(HOST_TEST_SESSION) + const before = await host.journalSnapshot(HOST_TEST_SESSION) rewind.mockRejectedValueOnce(new Error('read failed before revert')) - await expect(host.rewind(caller, params(target))).rejects.toThrow('read failed') + await expect(host.rewind(caller, await params(target))).rejects.toThrow('read failed') recoverRewind.mockResolvedValueOnce({ ok: false, reason: 'provider-refused' }) expect( await host.attach( @@ -328,9 +336,9 @@ describe('host rewind', () => { hostTestAttachParams(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) ) ).toMatchObject({ ok: true }) - expect(host.journalSnapshot(HOST_TEST_SESSION)).toEqual(before) + expect(await host.journalSnapshot(HOST_TEST_SESSION)).toEqual(before) expect(store.getRecord(HOST_TEST_SESSION)?.rewind?.phase).toBe('refused') - expect(await host.rewind(caller, params(target))).toMatchObject({ ok: true }) + expect(await host.rewind(caller, await params(target))).toMatchObject({ ok: true }) }) it('keeps host-stamped turn and goal rows through a Codex provider hydration', async () => { @@ -384,12 +392,17 @@ describe('host rewind', () => { return { ok: true, items } }) - expect(await host.rewind(caller, params(agentJournalItemKey(message('drop'))))).toMatchObject({ + expect( + await host.rewind(caller, await params(agentJournalItemKey(message('drop')))) + ).toMatchObject({ ok: true }) expect( - host.journalSnapshot(HOST_TEST_SESSION).items.map(({ itemId, body }) => ({ itemId, body })) + (await host.journalSnapshot(HOST_TEST_SESSION)).items.map(({ itemId, body }) => ({ + itemId, + body + })) ).toEqual([ { itemId: agentJournalItemKey(message('kept')), body: hostTestMessage('kept from provider') }, { itemId: agentJournalItemKey(goalRow), body: goalBody }, @@ -429,9 +442,9 @@ describe('host rewind', () => { throw new Error('lost after provider revert') }) - await expect(host.rewind(caller, params(agentJournalItemKey(message('drop'))))).rejects.toThrow( - 'lost after provider revert' - ) + await expect( + host.rewind(caller, await params(agentJournalItemKey(message('drop')))) + ).rejects.toThrow('lost after provider revert') recoverRewind.mockResolvedValueOnce({ ok: true, items: [{ identity: message('kept'), body: hostTestMessage('kept from recovery') }] @@ -444,7 +457,10 @@ describe('host rewind', () => { ).toMatchObject({ ok: true }) expect( - host.journalSnapshot(HOST_TEST_SESSION).items.map(({ itemId, body }) => ({ itemId, body })) + (await host.journalSnapshot(HOST_TEST_SESSION)).items.map(({ itemId, body }) => ({ + itemId, + body + })) ).toEqual([ { itemId: agentJournalItemKey(message('kept')), body: hostTestMessage('kept from recovery') }, { itemId: agentJournalItemKey(goalRow), body: goalBody } @@ -463,7 +479,7 @@ describe('host rewind', () => { await input.onReverted?.() throw new Error('lost after revert') }) - await expect(host.rewind(caller, params(target))).rejects.toThrow('lost after revert') + await expect(host.rewind(caller, await params(target))).rejects.toThrow('lost after revert') recoverRewind.mockResolvedValueOnce({ ok: true, items }) expect( await host.attach( @@ -471,7 +487,7 @@ describe('host rewind', () => { hostTestAttachParams(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) ) ).toMatchObject({ ok: true }) - expect(host.journalSnapshot(HOST_TEST_SESSION).items).toHaveLength(2) + expect((await host.journalSnapshot(HOST_TEST_SESSION)).items).toHaveLength(2) expect(store.getRecord(HOST_TEST_SESSION)?.rewind?.phase).toBe('completed') }) @@ -479,7 +495,7 @@ describe('host rewind', () => { 'never commits a recovered prefix that omits an expected retained %s', async (missing) => { const target = await seed() - const before = host.journalSnapshot(HOST_TEST_SESSION) + const before = await host.journalSnapshot(HOST_TEST_SESSION) const items = [0, 1].map((ordinal) => ({ identity: { provider: 'codex' as const, @@ -493,7 +509,7 @@ describe('host rewind', () => { await input.onPrepared?.(items) throw new Error('reply lost') }) - await expect(host.rewind(caller, params(target))).rejects.toThrow('reply lost') + await expect(host.rewind(caller, await params(target))).rejects.toThrow('reply lost') recoverRewind.mockResolvedValueOnce({ ok: true, items: missing === 'turn' ? [] : items.slice(0, 1) @@ -514,7 +530,7 @@ describe('host rewind', () => { it('settles the existing epoch after a crash between journal commit and record completion', async () => { const target = await seed() - const request = params(target) + const request = await params(target) const transition = store.transitionHandoff.bind(store) const checkpoint = vi .spyOn(store, 'transitionHandoff') @@ -528,7 +544,7 @@ describe('host rewind', () => { }) ) await expect(host.rewind(caller, request)).rejects.toThrow('completion write failed') - const committed = host.journalSnapshot(HOST_TEST_SESSION) + const committed = await host.journalSnapshot(HOST_TEST_SESSION) expect(committed.cursor.epoch).not.toBe(request.expectedEpoch) checkpoint.mockRestore() const replace = vi.spyOn(AgentSessionJournal.prototype, 'replaceEpochItems') @@ -538,7 +554,7 @@ describe('host rewind', () => { hostTestAttachParams(store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence) ) ).toMatchObject({ ok: true }) - expect(host.journalSnapshot(HOST_TEST_SESSION)).toEqual(committed) + expect(await host.journalSnapshot(HOST_TEST_SESSION)).toEqual(committed) expect(replace).not.toHaveBeenCalled() replace.mockRestore() expect(await host.rewind(caller, request)).toMatchObject({ ok: true, replayed: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index baedbe7c4d19..45dd7244896e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -13,6 +13,7 @@ import type { import type { AgentSessionMutationResult } from '../../../shared/agent-session-wire' import { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from './agent-session-history-page-bounds' import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' +import { openWithAgent } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionCaller } from './structured-agent-session-host-types' import { admitAndRunAgentSessionMutation } from './structured-agent-session-mutation-admission' @@ -35,6 +36,8 @@ export async function rewindStructuredAgentSession( adapter: context.deps.adapter, callerKey: caller.callerKey, envelope: params.envelope, + // Only the provider can do this, so an agent at rest is started first. + prepareSession: openWithAgent(context, params.envelope), journal: () => context.sessions.get(sessionId)?.journal, publish: (journal) => context.publish(sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts index 0f75c9a33224..2494535e019b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts @@ -120,11 +120,16 @@ describe('structured send idempotency', () => { await performSend(context, input) const replay = await performSend(context, input) + // Acceptance records the one submission; the reused id answers with it and writes nothing. + // Handing it over is the delivery loop's, never a second accept's. expect(replay).toMatchObject({ ok: true, - value: { clientMessageId: 'shared-send-id', submission: { dispatchState: 'accepted' } } + value: { + clientMessageId: 'shared-send-id', + submission: { dispatchState: 'pending', handoverRecorded: true } + } }) - expect(dispatch).toHaveBeenCalledOnce() + expect(dispatch).not.toHaveBeenCalled() expect(journal.submissions()).toHaveLength(1) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts index c23083fa687d..ba466fc3bc6e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts @@ -1,11 +1,11 @@ -// A send, or a hold, that finds the session's provider child gone, against the real host. +// A send, or a hold, that finds the session's provider child gone, against the real host. The +// send is accepted at once; its delivery restarts the child, or rejects it with the reason. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' -import { agentSessionRefusalOperationState } from '../../../shared/agent-session-refusal-retry' import type { AgentSessionMutationEnvelope, AgentSessionSubscribeEvent @@ -24,8 +24,11 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } -// Long enough that no release fires mid-test; whether one is pending is asserted directly. -const GRACE_MS = 60_000 + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} let root: string let store: AgentSessionRecordStore @@ -78,7 +81,6 @@ beforeEach(async () => { journalRoot: root, claimKeyId: 'key-1', mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, - releaseGraceMs: GRACE_MS, now: () => NOW, onEventSinkError: ({ error }) => hostErrors.push(error) }) @@ -105,15 +107,37 @@ function sendParams(text: string, operationId = hostTestOperationId()) { return { envelope, body } } -/** Every status row the chat shows, oldest first; none when the session is not even readable. */ -function journalStatuses(): string[] { - if (!host.hasSession(SESSION)) { - return [] - } - const history = host.history({ sessionId: SESSION, direction: 'tail' }) - return history.ok - ? history.page.items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) - : [] +/** Accepted at once, before any owner exists for it; answers the message id. */ +async function accept(params: ReturnType): Promise { + const result = await host.send(CALLER, params) + expect(result, JSON.stringify(result)).toMatchObject({ + ok: true, + replayed: false, + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } + }) + return params.envelope.clientOperationId +} + +async function submission(clientMessageId: string) { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) +} + +/** The submission once delivery is done with it: handed over, or rejected unwritten. */ +async function settled(clientMessageId: string) { + await eventually(async () => { + const current = await submission(clientMessageId) + expect(current?.dispatchState !== 'pending' || current.handedOverAt !== undefined).toBe(true) + }) + return submission(clientMessageId) +} + +/** The failure rows a start the chat needed left, oldest first. */ +async function errorStatuses(): Promise { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) } /** The child timed out or exited: its lease is handed back and the host holds no session. */ @@ -130,15 +154,14 @@ describe('a send with no live owner', () => { it('restarts the owner once and delivers against it', async () => { await loseOwner() - const result = await host.send(CALLER, sendParams('after the child died')) + await accept(sendParams('after the child died')) - expect(result).toMatchObject({ ok: true, replayed: false }) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') }) - it('restarts the owner before the send is admitted, so the send is admitted once', async () => { + it('accepts the send before anything restarts, and the restart hands it over', async () => { await loseOwner() const order: string[] = [] const spawnChild = acquire.getMockImplementation()! @@ -152,21 +175,18 @@ describe('a send with no live owner', () => { return admit(args) }) - await expect(host.send(CALLER, sendParams('ensure first'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('accept first')) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) - expect(order).toEqual(['acquire', 'admit']) + expect(order).toEqual(['admit', 'acquire']) }) it('leaves a live owner alone', async () => { acquire.mockClear() - await expect(host.send(CALLER, sendParams('owner is live'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('owner is live')) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).not.toHaveBeenCalled() }) @@ -184,37 +204,12 @@ describe('a send with no live owner', () => { } })) - await host.send(CALLER, sendParams('into a cleared chat')) - - expect(acquire).not.toHaveBeenCalled() - }) - - it('renews the idle window on journal activity in an unheld session', async () => { - await loseOwner() - await expect(host.send(CALLER, sendParams('restart'))).resolves.toMatchObject({ ok: true }) - const arm = vi.spyOn(host['holds']['clock'], 'arm') - - await expect(host.send(CALLER, sendParams('more activity'))).resolves.toMatchObject({ - ok: true - }) - - expect(arm).toHaveBeenCalledWith(SESSION) - }) - - it('releases the restarted child on the usual clock only when no surface holds it', async () => { - await loseOwner() - await expect(host.send(CALLER, sendParams('nobody is watching'))).resolves.toMatchObject({ - ok: true + await expect(host.send(CALLER, sendParams('into a cleared chat'))).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_operation_invalid' } }) - expect(host['holds'].isReleasePending(SESSION)).toBe(true) - await host.close(SESSION) - // A reading surface that does not itself restart the agent. - await host.hold(SESSION, 'desktop-chat:1', { resume: false }) - await expect(host.send(CALLER, sendParams('the chat is open'))).resolves.toMatchObject({ - ok: true - }) - expect(host['holds'].isReleasePending(SESSION)).toBe(false) + expect(acquire).not.toHaveBeenCalled() }) it('restarts an owner that exited while the session stayed readable', async () => { @@ -231,15 +226,15 @@ describe('a send with no live owner', () => { expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') acquire.mockClear() - await expect(host.send(CALLER, sendParams('after an exit'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('after an exit')) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() }) it('restarts nothing for a resend the journal already answers', async () => { const params = sendParams('sent once') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + await accept(params) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) // The child died during startup: the lease is handed back, the fence moves, and the session // stays readable. The client resends against the new fence. await host.handleAdapterEvent({ @@ -268,12 +263,9 @@ describe('a send with no live owner', () => { expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') // Retry rotates the id: a genuinely new send restarts the owner once. - await expect(host.send(CALLER, sendParams('sent once'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('sent once')) + await eventually(async () => expect(dispatch).toHaveBeenCalledTimes(2)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) }) it('restarts nothing for a send the ledger holds but the journal never saw', async () => { @@ -320,7 +312,7 @@ describe('a send with no live owner', () => { expect(dispatch).not.toHaveBeenCalled() }) - it('admits a send that arrives after the restart has already claimed the lease', async () => { + it('accepts a send that arrives while a restart holds the queue, and hands both over in order', async () => { await loseOwner() const lostFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 let claimed = () => {} @@ -334,18 +326,22 @@ describe('a send with no live owner', () => { return spawnChild!(input) }) - const first = host.send(CALLER, sendParams('first')) + const first = await accept(sendParams('first')) await claim expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(lostFence + 1) + // Written against the lost owner's fence, which admits it: a send is a conversation write. const late = sendParams('second') late.envelope.expectedRuntimeFence = lostFence const second = host.send(CALLER, late) release() - expect(await first).toMatchObject({ ok: true }) expect(await second).toMatchObject({ ok: true }) + await eventually(async () => expect(dispatch).toHaveBeenCalledTimes(2)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([ + first, + late.envelope.clientOperationId + ]) }) it('shares one restart between concurrent sends', async () => { @@ -358,142 +354,108 @@ describe('a send with no live owner', () => { ]) expect(results.map((result) => result.ok)).toEqual([true, true, true]) + await eventually(async () => expect(dispatch).toHaveBeenCalledTimes(3)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(3) - }) - - it('shares one restart between a hold and a send that arrive in the same gap', async () => { - await loseOwner() - - const [held, sent] = await Promise.allSettled([ - host.hold(SESSION, 'desktop-chat:1'), - host.send(CALLER, sendParams('while the chat opens')) - ]) - - expect(held).toMatchObject({ status: 'fulfilled' }) - expect(sent).toMatchObject({ status: 'fulfilled', value: { ok: true } }) - expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledOnce() - expect(host['holds'].isHeld(SESSION)).toBe(true) - expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) it('replays into a closed session without spawning anything', async () => { const params = sendParams('sent once') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + await accept(params) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) await loseOwner() await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) - // The journal was made readable for the answer; the record's lease was left as it was. + // The conversation was opened for the answer; the record's lease was left as it was. expect(host.hasSession(SESSION)).toBe(true) expect(acquire).not.toHaveBeenCalled() expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) - it("refuses with the restart's own cause, in the answer and in the chat", async () => { + it("rejects the accepted message with the restart's own cause, and says so in the chat once", async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in. Run codex login')) const params = sendParams('while signed out') + const cause = + 'The provider stopped before it finished starting: Not signed in. Run codex login.' - const result = await host.send(CALLER, params) + const id = await accept(params) - expect(result).toEqual({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: "Codex couldn't restart: Not signed in. Run codex login.", - // The failed attach proved its child gone: nothing runs for this session. - ownerVerdict: 'exited' - } - }) + expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause }) expect(dispatch).not.toHaveBeenCalled() - expect(hostErrors).not.toEqual([]) - expect(agentSessionRefusalOperationState('agent_session_owner_restart_failed')).toBe( - 'settled-rejected' - ) - // Refused before admission: the ledger holds nothing a resend would replay. - expect(store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId)).toBeNull() - // The same status row a failed start leaves, so the reason outlives the error strip. - expect(journalStatuses()).toEqual([ - 'The provider stopped before it finished starting: Not signed in. Run codex login.' - ]) + // Accepted, so the ledger answers a resend with the rejection rather than a second attempt. + expect( + store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId) + ).toMatchObject({ outcome: { status: 'succeeded' } }) + // One row, in the error tone, so the reason outlives the error strip. + expect(await errorStatuses()).toEqual([cause]) }) - it('restarts again for a Retry of the refused send, under its own id or a new one', async () => { + it('restarts again for a Retry under a new id, and replays a resend of the same id', async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in')) const params = sendParams('while signed out') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + await settled(await accept(params)) + expect(acquire).toHaveBeenCalledTimes(1) - // A client that resends the same id gets another attempt, and the chat no second row. + // A client that resends the same id gets the recorded rejection, and the chat no second row. await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } + ok: true, + replayed: true, + value: { submission: { dispatchState: 'rejected' } } }) - expect(acquire).toHaveBeenCalledTimes(2) - expect(journalStatuses()).toHaveLength(1) + expect(acquire).toHaveBeenCalledTimes(1) + expect(await errorStatuses()).toHaveLength(1) - // The outbox's Retry rotates the id: also a fresh attempt. - await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } + // The outbox's Retry rotates the id: a fresh attempt, with its own row. + expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({ + dispatchState: 'rejected' }) - expect(acquire).toHaveBeenCalledTimes(3) + expect(acquire).toHaveBeenCalledTimes(2) + expect(await errorStatuses()).toHaveLength(2) expect(dispatch).not.toHaveBeenCalled() }) it('restarts and delivers a later send once the cause clears', async () => { await loseOwner() acquire.mockRejectedValueOnce(new Error('Not signed in')) - await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } + expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({ + dispatchState: 'rejected' }) // The user signed in; nothing about the failed attempt is remembered. - await expect(host.send(CALLER, sendParams('signed in now'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('signed in now')) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledTimes(2) - expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') }) it('suggests a new chat only when this host has nothing to restart the chat from', async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in')) - const failed = await host.send(CALLER, sendParams('restart fails')) - expect(failed).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) - expect(failed.ok ? '' : failed.refusal.message).not.toMatch(/new chat/) + const failed = await settled(await accept(sendParams('restart fails'))) + expect(failed).toMatchObject({ dispatchState: 'rejected' }) + expect(failed?.reason).not.toMatch(/new chat/) // The adapter cannot run this record where it lives: no retry would bring it back. host.deps.adapter.supportsLocation = () => false - const unresumable = await host.send(CALLER, sendParams('cannot resume here')) + const unresumable = await settled(await accept(sendParams('cannot resume here'))) expect(unresumable).toMatchObject({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: - "Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue." - } + dispatchState: 'rejected', + reason: + "Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue." }) }) - it('runs the send as the lease stands when the restart met a lease someone else is settling', async () => { + it('rejects the message with the cause when the restart met a lease someone else is settling', async () => { await loseOwner() - vi.spyOn(host['holds'], 'ensureProviderChild').mockResolvedValueOnce({ + vi.spyOn( + host['conversationDelivery'].loop['deps'], + 'ensureProviderChild' + ).mockResolvedValueOnce({ ok: false, refusal: { code: 'execution_owner_reconciling', @@ -501,151 +463,34 @@ describe('a send with no live owner', () => { } }) - const result = await host.send(CALLER, sendParams('owner being settled')) + const id = await accept(sendParams('owner being settled')) - // The ordinary lease check answers, retryably; nothing terminal and nothing in the chat. - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } - }) + const cause = "Codex couldn't restart: Another runtime is still adjudicating this lease." + expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause }) expect(acquire).not.toHaveBeenCalled() - expect(journalStatuses()).toEqual([]) + expect(await errorStatuses()).toEqual([cause]) }) - it('runs the send as the lease stands when the restart itself faults', async () => { + it('rejects the message, and reports the fault, when the restart itself faults', async () => { await loseOwner() - vi.spyOn(host['holds'], 'ensureProviderChild').mockRejectedValueOnce( - new Error('spawn-token mint failed') - ) + vi.spyOn( + host['conversationDelivery'].loop['deps'], + 'ensureProviderChild' + ).mockRejectedValueOnce(new Error('spawn-token mint failed')) - const result = await host.send(CALLER, sendParams('bookkeeping failed')) + const id = await accept(sendParams('bookkeeping failed')) - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } + expect(await settled(id)).toMatchObject({ + dispatchState: 'rejected', + reason: "Codex couldn't restart: spawn-token mint failed." }) expect(hostErrors).toContainEqual( expect.objectContaining({ message: 'spawn-token mint failed' }) ) - expect(journalStatuses()).toEqual([]) - }) - - it('keeps a second surface holder taken during an auto-restart, and starts nothing for it', async () => { - await host.hold(SESSION, 'desktop-chat:1') - const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 - acquire.mockClear() - const entered = Promise.withResolvers() - const gate = Promise.withResolvers() - const spawnChild = acquire.getMockImplementation()! - acquire.mockImplementationOnce(async (input) => { - entered.resolve() - await gate.promise - return spawnChild(input) - }) - - // The held child exits: the host restarts it on its own, under the surface that holds it. - const restarted = host.handleAdapterEvent({ - type: 'ended', - sessionId: SESSION, - reason: 'provider exited', - cause: 'unexpected-exit', - fence: exitedFence, - acquisitionGeneration: 'generation-1' - }) - await entered.promise - const second = host.hold(SESSION, 'paired-phone:1') - gate.resolve() - await Promise.all([restarted, second]) - - expect(acquire).toHaveBeenCalledOnce() - expect(host['holds']['holders'].holderIds(SESSION)).toEqual([ - 'desktop-chat:1', - 'paired-phone:1' - ]) - expect(host['holds'].isReleasePending(SESSION)).toBe(false) - expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + expect(await errorStatuses()).toHaveLength(1) }) - it('puts the child of an auto-restart on the idle clock when its surface left mid-attach', async () => { - await host.hold(SESSION, 'desktop-chat:1') - const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 - acquire.mockClear() - const entered = Promise.withResolvers() - const gate = Promise.withResolvers() - const spawnChild = acquire.getMockImplementation()! - acquire.mockImplementationOnce(async (input) => { - entered.resolve() - await gate.promise - return spawnChild(input) - }) - - const restarted = host.handleAdapterEvent({ - type: 'ended', - sessionId: SESSION, - reason: 'provider exited', - cause: 'unexpected-exit', - fence: exitedFence, - acquisitionGeneration: 'generation-1' - }) - await entered.promise - // The only surface leaves while the host is still spawning the child it asked for. - host.release(SESSION, 'desktop-chat:1') - gate.resolve() - await restarted - - expect(acquire).toHaveBeenCalledOnce() - expect(host['holds'].isHeld(SESSION)).toBe(false) - // Nobody holds the child, so it goes on the same clock a departed surface would start. - expect(host['holds'].isReleasePending(SESSION)).toBe(true) - expect(hostErrors).toEqual([]) - }) - - it('counts a queued restart as in flight from the moment it is asked for, so a quit drains it', async () => { - const closeSession = vi.mocked(host.deps.adapter.closeSession!) - acquire.mockClear() - const gate = Promise.withResolvers() - closeSession.mockImplementationOnce(async () => { - await gate.promise - return true - }) - const closing = host.close(SESSION) - const hold = host.hold(SESSION, 'desktop-chat:1') - let drained = false - void host['tasks'].drainAttaches().then(() => { - drained = true - }) - await new Promise((resolve) => setImmediate(resolve)) - - // The hold waits its turn behind the close, and the quit's drain waits for the hold: the - // child it is about to spawn must exist before the quit decides what to evict. - expect(acquire).not.toHaveBeenCalled() - expect(drained).toBe(false) - - gate.resolve() - await Promise.all([closing, hold]) - await new Promise((resolve) => setImmediate(resolve)) - expect(acquire).toHaveBeenCalledOnce() - expect(drained).toBe(true) - }) - - it('adjudicates a lease this host has not reconciled before a hold resumes it', async () => { - await loseOwner() - await store.transitionHandoff(SESSION, (current) => ({ - ...current, - lease: { ...current.lease, unreconciled: true } - })) - host.deps.probeOwner = async () => ({ outcome: 'pid-absent' }) - - await host.hold(SESSION, 'desktop-chat:1') - - expect(acquire).toHaveBeenCalledOnce() - expect(store.getRecord(SESSION)?.lease).toMatchObject({ - unreconciled: false, - claimStatus: 'live' - }) - }) - - it('exits the recovery stage a failed attempt latched before a hold resumes', async () => { + it('exits the recovery stage a failed attempt latched before its delivery resumes it', async () => { await loseOwner() // What an acquisition whose exit could not be proven leaves behind: nobody's, but latched. await store.transitionHandoff(SESSION, (current) => ({ @@ -654,8 +499,9 @@ describe('a send with no live owner', () => { })) host.deps.probeOwner = async () => ({ outcome: 'pid-absent' }) - await host.hold(SESSION, 'desktop-chat:1') + await accept(sendParams('latched owner')) + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ handoffStage: null, @@ -663,20 +509,22 @@ describe('a send with no live owner', () => { }) }) - it('leaves a lease it cannot adjudicate alone', async () => { + it('adjudicates a lease this host has not reconciled before its delivery resumes it', async () => { await loseOwner() await store.transitionHandoff(SESSION, (current) => ({ ...current, lease: { ...current.lease, unreconciled: true } })) - const result = await host.send(CALLER, sendParams('owner unverifiable')) + // The send's start is the same serialized resume a hold runs, reconciliation first. + await accept(sendParams('owner unverified')) - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } + await eventually(async () => expect(dispatch).toHaveBeenCalledOnce()) + expect(acquire).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + unreconciled: false, + claimStatus: 'live' }) - expect(acquire).not.toHaveBeenCalled() }) }) @@ -689,32 +537,30 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { const params = sendParams('after the release') params.envelope.expectedRuntimeFence = seenFence - expect(await host.send(CALLER, params)).toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } - }) + const id = await accept(params) + expect(await settled(id)).toMatchObject({ dispatchState: 'accepted' }) expect(acquire).toHaveBeenCalledOnce() expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(seenFence + 1) }) - // Clients resend a refused message when the fence they hold moves, and a refused restart leaves - // no ledger row, so a frame carrying each failed attempt's fence would resend it forever. - it("keeps the pane's fence on the row a failed restart publishes", async () => { + // Clients resend a refused message when the fence they hold moves. A failed start is a + // rejected message now, never a refused send, and the pane keeps the fence it subscribed under. + it("keeps the pane's fence on the rows a failed start publishes, and rejects the message", async () => { const seenFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 const frames: AgentSessionSubscribeEvent[] = [] - host.subscribe({ id: 'pane', sessionId: SESSION, emit: (event) => frames.push(event) }) + await host.subscribe({ id: 'pane', sessionId: SESSION, emit: (event) => frames.push(event) }) await loseOwner() acquire.mockRejectedValueOnce(new Error('Not signed in')) const subscribed = frames.length - expect(await host.send(CALLER, sendParams('while signed out'))).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + const id = await accept(sendParams('while signed out')) + expect(await settled(id)).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining('Not signed in') + }) expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(seenFence + 1) const published = frames.slice(subscribed) expect(published.length).toBeGreaterThan(0) @@ -736,7 +582,8 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { return spawnChild(input) }) - const first = host.send(CALLER, sendParams('starts the agent')) + const firstParams = sendParams('starts the agent') + const first = host.send(CALLER, firstParams) await claim const cancelFields = { turnId: 'turn-1' } const stop = host.cancel(CALLER, { @@ -760,7 +607,14 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { expect(await first).toMatchObject({ ok: true }) expect(await stop).toMatchObject({ ok: true, replayed: false }) expect(await second).toMatchObject({ ok: true, replayed: false }) + // The Stop withdrew the message its start held; the one typed after it is delivered. + expect(await settled(late.envelope.clientOperationId)).toMatchObject({ + dispatchState: 'accepted' + }) + expect(await submission(firstParams.envelope.clientOperationId)).toMatchObject({ + dispatchState: 'rejected' + }) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) + expect(dispatch).toHaveBeenCalledTimes(1) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts index 09986f2e1296..d95369a04947 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts @@ -1,23 +1,8 @@ -// What a send needs from the session before its lease is checked. -// -// A provider child that exits or fails to start hands its lease back. Before this, a send to that -// session was refused `agent_session_ownership_unknown` — which a client reads as "not admitted -// yet" and resends forever — and only a surface hold could ever make a new child. Now the send -// makes sure it has an owner as a step of its own serialized admission: a released lease where -// resume is allowed gets a child first; anything else runs as it is and meets the lease check. -// A restart that fails refuses with a code the client stops auto-retrying on, carrying the -// restart's own cause, and writes that cause into the chat the way a start that failed does, so -// the user sees why. A manual Retry or a new send is a fresh attempt: a refusal before admission -// leaves no ledger row behind. -// -// The ledger's answer comes first, so a send it already holds a row for restarts nothing: -// admission replays or refuses it whoever owns the session now, and a closed session is made -// readable for that, never given a child. Otherwise each resend of a message whose child died at -// startup would spawn another child that dies the same way. -// -// A child that has not proven its start is still the owner: the send is admitted against it and -// the adapter holds the message until startup lands, or rejects it with the child's own reason -// when the child dies first. The exit settlement writes that reason into the chat. +// What a send or a Stop needs from the session before the ledger places its row: the +// conversation open. Nothing here needs an owner — a send is accepted into the conversation and +// the delivery loop makes the session ready — so a refusal before acceptance is only one the +// conversation itself makes: a rewind or conversation command in doubt, a cleared conversation, +// or a journal that cannot be opened. import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { @@ -25,45 +10,40 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { AgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals' -import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds' import { TUI_AGENT_DISPLAY_NAMES } from '../../../shared/tui-agent-display-names' import { ownerRestartFailedOutcome, providerStartupFailureOutcome } from './structured-agent-session-dead-generation-settlement' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' -import type { AgentSessionMutationSessionPreparation } from './structured-agent-session-mutation-admission' -import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' +import { + AGENT_SESSION_NOT_ATTACHED, + type AgentSessionMutationSessionPreparation +} from './structured-agent-session-mutation-admission' import { rewindRefusal } from './structured-rewind-refusal' +import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' /** - * What a refused resume means for the send that ran it. `transient`: the resume met a lease - * someone else is settling, which is not proof it cannot resume — the send runs as the lease - * stands and admission reports it. `failed`: the restart itself failed; the send answers with the - * cause and stops the client's retry loop, and the user may clear the cause and retry. - * `unresumable`: this host has nothing to restart the chat from — no record, or none it can run — - * so only a new chat continues. A new wire code does not compile until it is classified here. + * Whether a refused start leaves the chat anything to start again from. `unresumable`: this host + * has nothing to restart it from — no record, or none it can run — so only a new chat continues. + * A new wire code does not compile until it is classified here. */ -const RESUME_REFUSAL_OUTCOME: Record< - AgentSessionWireRefusalCode, - 'transient' | 'failed' | 'unresumable' -> = { - execution_owner_reconciling: 'transient', - agent_session_conflict: 'transient', - agent_session_checkpoint_stale: 'transient', - agent_session_ownership_unknown: 'transient', - agent_session_operation_capacity: 'transient', - structured_agent_session_unsupported: 'unresumable', - agent_session_operation_conflict: 'failed', - agent_session_operation_expired: 'failed', - agent_session_operation_invalid: 'failed', - agent_session_operation_unknown: 'failed', - agent_session_item_revision_stale: 'failed', - agent_session_already_resolved: 'failed', - agent_session_identity_required: 'unresumable', - agent_session_journal_unreadable: 'failed', - agent_session_owner_restart_failed: 'failed' +const START_REFUSAL_RESUMABLE: Record = { + execution_owner_reconciling: true, + agent_session_conflict: true, + agent_session_checkpoint_stale: true, + agent_session_ownership_unknown: true, + agent_session_operation_capacity: true, + structured_agent_session_unsupported: false, + agent_session_operation_conflict: true, + agent_session_operation_expired: true, + agent_session_operation_invalid: true, + agent_session_operation_unknown: true, + agent_session_item_revision_stale: true, + agent_session_already_resolved: true, + agent_session_identity_required: false, + agent_session_journal_unreadable: true, + agent_session_owner_restart_failed: true } /** Why the record refuses any send right now, whoever owns it; null when a send may run. */ @@ -93,133 +73,75 @@ export function structuredAgentSessionSendBlock( return null } -/** Whether this send is the one that must bring the owner back: no child, a lease handed back - * cleanly, and nothing on the record that refuses the send anyway. Live, unverifiable, still - * reserved, or handed off: that lease is not this send's to replace. */ -export function structuredAgentSessionSendNeedsOwner( - session: StructuredAgentSessionHostSession | undefined, - record: AgentSessionRecord -): boolean { - return ( - session?.hasProviderChild !== true && - isResumableStructuredAgentSessionRecord(record) && - structuredAgentSessionSendBlock(record) === null - ) -} - -type SendPreparationContext = Pick< - StructuredAgentSessionMutationContext, - 'deps' | 'sessions' | 'holds' | 'restoreReadable' -> - -export async function prepareStructuredAgentSessionSend( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - ledger: 'admit' | 'replay', - record: AgentSessionRecord +/** The conversation a send or a Stop writes to, opened when this host holds it closed. */ +export async function openConversationForWrite( + openConversation: (sessionId: string) => Promise, + envelope: AgentSessionMutationEnvelope ): Promise { - const { sessionId } = record - if (ledger !== 'admit') { - if (!context.sessions.has(sessionId)) { - await context.restoreReadable(sessionId) + try { + if (await openConversation(envelope.sessionId)) { + return { ok: true } } - return { ok: true } - } - if (structuredAgentSessionSendNeedsOwner(context.sessions.get(sessionId), record)) { - const refusal = await restartOwnerForSend(context, envelope, record) - if (refusal) { - return { ok: false, refusal } + return { ok: false, refusal: AGENT_SESSION_NOT_ATTACHED } + } catch (error) { + return { + ok: false, + refusal: { + code: 'agent_session_journal_unreadable', + message: `The conversation could not be opened: ${ + error instanceof Error ? error.message : String(error) + }` + } } } - return { ok: true } } -/** One restart attempt. Answers with the refusal that ends the send, or null when the send goes - * on to admission — after a child, after a transient refusal, or after a fault in the restart's - * own bookkeeping, which is reported and never gates the user's action. */ -async function restartOwnerForSend( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - record: AgentSessionRecord -): Promise { - const { sessionId } = envelope - let resumed: Awaited> - try { - resumed = await context.holds.ensureProviderChild(sessionId) - } catch (error) { - context.deps.onEventSinkError?.({ sessionId, error }) - return null - } - const outcome = resumed.ok ? null : RESUME_REFUSAL_OUTCOME[resumed.refusal.code] - if (resumed.ok || outcome === 'transient') { - return null +/** The conversation a write lands in, opened when this host holds it closed. */ +export function openForWrite( + context: Pick, + envelope: AgentSessionMutationEnvelope +): () => Promise { + return () => openConversationForWrite(context.openConversation, envelope) +} + +/** For an operation only the provider can perform: the conversation, then its agent. */ +export function openWithAgent( + context: Pick, + envelope: AgentSessionMutationEnvelope +): () => Promise { + return async () => { + const opened = await openConversationForWrite(context.openConversation, envelope) + return opened.ok ? context.ensureAgent(envelope.sessionId) : opened } - const refusal = ownerRestartFailedRefusal(record, resumed.refusal, outcome !== 'unresumable') - context.deps.onEventSinkError?.({ - sessionId, - error: new Error(`${resumed.refusal.code}: ${resumed.refusal.message}`) - }) - // A restart whose child died starting leaves the row any start that died leaves, so the chat - // reads the same whether the send met that death before admission or after it. - await recordFailedRestart( - context, - envelope, - resumed.refusal.ownerVerdict === 'exited' - ? providerStartupFailureOutcome(resumed.refusal.message) - : refusal.message - ) - return refusal } -/** The client stops on the code; the message carries the restart's own cause, and the verdict — - * when the failed attach proved its child gone — tells a client nothing runs for the session. */ -function ownerRestartFailedRefusal( - record: AgentSessionRecord, - cause: AgentSessionWireRefusal, - resumable: boolean -): AgentSessionWireRefusal { - return { - code: 'agent_session_owner_restart_failed', - message: ownerRestartFailedOutcome({ - agentName: TUI_AGENT_DISPLAY_NAMES[record.provider], - reason: cause.message, - resumable - }), - ...(cause.ownerVerdict ? { ownerVerdict: cause.ownerVerdict } : {}) +/** A rewind still in doubt once the conversation is open is one only its provider can settle — + * the open settles every other — so a send starts the agent, whose attach recovers it. */ +export function sendPreparation( + context: Pick, + envelope: AgentSessionMutationEnvelope +): () => Promise { + return async () => { + const opened = await openConversationForWrite(context.openConversation, envelope) + const phase = context.deps.store.getRecord(envelope.sessionId)?.rewind?.phase + return opened.ok && (phase === 'prepared' || phase === 'provider-succeeded') + ? context.ensureAgent(envelope.sessionId) + : opened } } -/** The same status row a start that failed leaves in the chat, so the reason outlives the error - * strip. The journal is made readable for it when the failed attach left none behind. Keyed by - * the send, not the clock: a resend of the same id that fails again adds no second row. */ -async function recordFailedRestart( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - text: string -): Promise { - const { sessionId } = envelope - try { - if (!context.sessions.has(sessionId)) { - await context.restoreReadable(sessionId) - } - const session = context.sessions.get(sessionId) - if (!session) { - return - } - const settlementId = `failed-restart:${envelope.clientOperationId}` - await session.journal.appendLifecycleBatch({ - settlementId, - fence: session.fence, - recovered: true, - mutations: [ - { - kind: 'item', - identity: { provider: 'orca', clientMessageId: settlementId }, - body: { kind: 'status', text: boundJournalStatusText(text) } - } - ] - }) - } catch (error) { - context.deps.onEventSinkError?.({ sessionId, error }) +/** What the chat says, in its row and on every message it rejects, when the delivery loop could + * not make the session ready. A child that died starting reads as any start that died does. */ +export function structuredAgentSessionStartFailureText( + record: AgentSessionRecord | null, + cause: AgentSessionWireRefusal +): string { + if (cause.ownerVerdict === 'exited') { + return providerStartupFailureOutcome(cause.message) } + return ownerRestartFailedOutcome({ + agentName: record ? TUI_AGENT_DISPLAY_NAMES[record.provider] : 'The agent', + reason: cause.message, + resumable: START_REFUSAL_RESUMABLE[cause.code] + }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts index e2e2fb81102e..e1bb80e84599 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts @@ -1,12 +1,11 @@ // A session that published and then lost its child before startup (not signed in, say) keeps a // released lease and a chat the user can still type into. The send is the user asking for the -// child back: the host restarts it before admitting the write and delivers against the new owner, -// instead of parking the message behind a lease nothing would ever re-acquire. +// child back: the host accepts the message, and its delivery restarts the child and hands the +// message to the new owner, instead of parking it behind a lease nothing would ever re-acquire. // // A child is published before it has proven its start, and it owns the send from that moment: the -// message is admitted against it and the adapter holds it for the start. When the child exits -// first, the exit settlement rejects the message and writes the cause into the chat, once, and -// the next send is a fresh restart. +// message is handed to it. When the child exits first, the exit settlement rejects the message and +// writes the cause into the chat, once, and the next send is a fresh restart. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -28,6 +27,11 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} const EXIT_REASON = 'Claude Code is not signed in. Sign in with the Claude CLI' let root: string @@ -53,7 +57,7 @@ function sendEnvelope( } } -/** A send is admitted against the child it meets, proven or not; the adapter holds the rest. */ +/** A send is accepted at once and handed to the child delivery finds or starts. */ async function send( text: string, fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -63,9 +67,13 @@ async function send( expect(sent, JSON.stringify(sent)).toMatchObject({ ok: true, replayed: false, - value: { submission: { dispatchState: 'pending' } } + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } }) - return sent.ok ? sent.value.clientMessageId : '' + const clientMessageId = sent.ok ? sent.value.clientMessageId : '' + await eventually(async () => + expect((await submission(clientMessageId))?.handedOverAt).toBeDefined() + ) + return clientMessageId } /** The child of the current acquisition, as the adapter would identify it in a lifecycle event. */ @@ -96,16 +104,16 @@ function exitBeforeProof(): Promise { }) } -function journalStatuses(): string[] { - return host - .journalSnapshot(SESSION) - .items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) +async function journalStatuses(): Promise { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' ? [item.body.text] : [] + ) } -function submission(clientMessageId: string) { - return host - .journalSnapshot(SESSION) - .submissions.find((entry) => entry.clientMessageId === clientMessageId) +async function submission(clientMessageId: string) { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) } beforeEach(async () => { @@ -166,8 +174,7 @@ describe('a send into a published session whose child ended before startup', () await send('hello again', releasedFence) - // A send still fenced to the lost owner is admitted once against the restarted one, with no - // stale round trip. + // Accepted at the lost owner's fence and handed to the child delivery started, once. expect(acquire).toHaveBeenCalledTimes(2) expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') expect(dispatch).toHaveBeenCalledOnce() @@ -183,22 +190,22 @@ describe('a send into a published session whose child ended before startup', () it('retires the held message with the cause when the restarted child exits before proving its start', async () => { const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 - const rowsBefore = journalStatuses().length + const rowsBefore = (await journalStatuses()).length const held = await send('still not signed in', releasedFence) await exitBeforeProof() // The child never proved its start, so it accepted nothing: the exit rejects the message this // host admitted, so nothing pins the session and Retry stays offered, and one row names the cause. - expect(submission(held)).toMatchObject({ + expect(await submission(held)).toMatchObject({ dispatchState: 'rejected', reason: expect.stringContaining(EXIT_REASON), recovered: true }) expect( - host.journalSnapshot(SESSION).submissions.filter((e) => e.dispatchState === 'pending') + (await host.journalSnapshot(SESSION)).submissions.filter((e) => e.dispatchState === 'pending') ).toEqual([]) - expect(journalStatuses().slice(rowsBefore)).toEqual([ + expect((await journalStatuses()).slice(rowsBefore)).toEqual([ expect.stringMatching(/stopped before it finished starting: .*not signed in/) ]) // The failed restart moved the fence twice: the acquisition, and the exit that released it. @@ -207,16 +214,16 @@ describe('a send into a published session whose child ended before startup', () // One spawn per user action: nothing restarted it a second time. expect(acquire).toHaveBeenCalledTimes(2) - // Retry is a fresh action: it restarts once and is admitted against the new child. + // Retry is a fresh action: it restarts once and is handed to the new child. await send('signed in now') expect(acquire).toHaveBeenCalledTimes(3) expect(dispatch).toHaveBeenCalledTimes(2) - expect(journalStatuses().slice(rowsBefore)).toHaveLength(1) + expect((await journalStatuses()).slice(rowsBefore)).toHaveLength(1) }) }) describe('a send while the child of the first start is still proving itself', () => { - it('is admitted against the starting child, and nothing restarts it', async () => { + it('is handed to the starting child, and nothing restarts it', async () => { await send('hello') expect(dispatch).toHaveBeenCalledOnce() @@ -225,7 +232,7 @@ describe('a send while the child of the first start is still proving itself', () await proveStarted() expect(acquire).toHaveBeenCalledOnce() - expect(journalStatuses()).toEqual([]) + expect(await journalStatuses()).toEqual([]) }) it('is retired with the cause when that child exits first, and restarts nothing', async () => { @@ -234,13 +241,13 @@ describe('a send while the child of the first start is still proving itself', () await exitBeforeProof() - expect(submission(held)).toMatchObject({ + expect(await submission(held)).toMatchObject({ dispatchState: 'rejected', reason: expect.stringContaining(EXIT_REASON), recovered: true }) expect(acquire).toHaveBeenCalledOnce() - expect(journalStatuses()).toEqual([ + expect(await journalStatuses()).toEqual([ expect.stringMatching(/stopped before it finished starting: .*not signed in/) ]) expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(fence + 1) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts index b461d508f42a..ff9765d1cee7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts @@ -61,7 +61,7 @@ describe('structured send settlement compatibility wait', () => { it('removes an abandoned wait on transport cancellation', async () => { const settlements = new StructuredAgentSessionSendSettlement(() => journal('pending')) const controller = new AbortController() - const pending = settlements.wait('session-1', 'client-1', controller.signal) + const pending = settlements.wait('session-1', 'client-1', { signal: controller.signal }) controller.abort(new Error('transport closed')) await expect(pending).rejects.toThrow('transport closed') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts index 6150e3412a69..bc265833c429 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts @@ -3,8 +3,12 @@ import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionSendResult } from '../../../shared/agent-session-wire' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +/** What a settlement read needs of a journal. */ +type SendSettlementJournal = Pick + type SettledSend = { cursor: AgentJournalCursor value: AgentSessionSendResult @@ -12,8 +16,19 @@ type SettledSend = { type SendSettlement = SettledSend | 'pending' | 'missing' +/** What ends a wait: the provider's answer, or only the host handing the message over. */ +export type SendSettlementPoint = 'answered' | 'handed-over' + +export type SendSettlementWaitOptions = { + signal?: AbortSignal + /** How long to observe; unanswered by then resolves undefined. */ + budgetMs?: number + until?: SendSettlementPoint +} + type SendSettlementWaiter = { clientMessageId: string + until: SendSettlementPoint resolve: (result: SettledSend | undefined) => void reject: (error: Error) => void timer: ReturnType @@ -23,12 +38,15 @@ type SendSettlementWaiter = { // Known legacy clients abandon the RPC after 15s without cancelling its socket dispatch. const SEND_SETTLEMENT_WAIT_TIMEOUT_MS = 30_000 +/** Long enough for a cold provider start; a longer one replays through the same wait. */ +export const STRUCTURED_AGENT_SESSION_START_WAIT_MS = 120_000 const MAX_SEND_SETTLEMENT_WAITERS_PER_SESSION = 64 const MAX_SEND_SETTLEMENT_WAITERS = 1_024 function settledSend( - journal: AgentSessionJournal, + journal: SendSettlementJournal, clientMessageId: string, + until: SendSettlementPoint, submission: AgentJournalSubmission | undefined = journal .submissions() .find((candidate) => candidate.clientMessageId === clientMessageId) @@ -36,9 +54,11 @@ function settledSend( if (!submission) { return 'missing' } - return submission.dispatchState === 'pending' - ? 'pending' - : { cursor: journal.cursor(), value: { clientMessageId, submission } } + const waiting = + until === 'handed-over' + ? isQueuedAgentJournalSubmission(submission) + : submission.dispatchState === 'pending' + return waiting ? 'pending' : { cursor: journal.cursor(), value: { clientMessageId, submission } } } function abortError(signal: AbortSignal): Error { @@ -52,17 +72,19 @@ export class StructuredAgentSessionSendSettlement { private readonly waiters = new Map>() private waiterCount = 0 - constructor(private readonly journalFor: (sessionId: string) => AgentSessionJournal) {} + constructor(private readonly journalFor: (sessionId: string) => SendSettlementJournal) {} wait = ( sessionId: string, clientMessageId: string, - signal?: AbortSignal + options: SendSettlementWaitOptions = {} ): Promise => { + const { signal } = options + const until = options.until ?? 'answered' if (signal?.aborted) { return Promise.reject(abortError(signal)) } - const immediate = settledSend(this.journalFor(sessionId), clientMessageId) + const immediate = settledSend(this.journalFor(sessionId), clientMessageId, until) if (immediate === 'missing') { return Promise.reject(new Error('agent session send disappeared before settlement')) } @@ -79,12 +101,13 @@ export class StructuredAgentSessionSendSettlement { return new Promise((resolve, reject) => { const waiter: SendSettlementWaiter = { clientMessageId, + until, resolve, reject, timer: setTimeout(() => { this.remove(sessionId, waiter) resolve(undefined) - }, SEND_SETTLEMENT_WAIT_TIMEOUT_MS) + }, options.budgetMs ?? SEND_SETTLEMENT_WAIT_TIMEOUT_MS) } waiter.timer.unref?.() const session = existingSession ?? new Set() @@ -106,7 +129,7 @@ export class StructuredAgentSessionSendSettlement { }) } - publish(sessionId: string, journal: AgentSessionJournal): void { + publish(sessionId: string, journal: SendSettlementJournal): void { const waiters = this.waiters.get(sessionId) if (!waiters) { return @@ -118,6 +141,7 @@ export class StructuredAgentSessionSendSettlement { const result = settledSend( journal, waiter.clientMessageId, + waiter.until, submissions.get(waiter.clientMessageId) ) if (result !== 'pending') { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts index 638d8a8679f6..24e55bbba636 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts @@ -32,6 +32,28 @@ beforeEach(() => { ;({ store, host, dispatch } = hostTestState()) }) +function hostJournal(): AgentSessionJournal { + return ( + host as unknown as { sessions: Map } + ).sessions.get(SESSION)!.journal +} + +/** A send is accepted, then the session's delivery loop hands it over: wait for the handover's + * outcome, or with `handedOver`, only for the handover itself (an admitted send stays pending). */ +async function delivered(clientMessageId: string, options: { handedOver?: true } = {}) { + let submission: ReturnType[number] | undefined + await vi.waitFor(() => { + submission = hostJournal() + .submissions() + .find((entry) => entry.clientMessageId === clientMessageId) + expect(submission?.handedOverAt).toBeDefined() + if (!options.handedOver) { + expect(submission?.dispatchState).not.toBe('pending') + } + }) + return submission! +} + describe('send', () => { it('writes the submission before dispatching and resolves it accepted', async () => { await attach() @@ -43,9 +65,17 @@ describe('send', () => { if (!result.ok) { throw new Error(`expected a send, got ${result.refusal.code}`) } - expect(result.value.submission.dispatchState).toBe('accepted') + // Answered once accepted; the delivery loop hands it over after. + expect(result.value.submission).toMatchObject({ + dispatchState: 'pending', + handoverRecorded: true + }) + expect(result.value.submission.handedOverAt).toBeUndefined() + await expect(delivered(result.value.clientMessageId)).resolves.toMatchObject({ + dispatchState: 'accepted' + }) expect(dispatch).toHaveBeenCalledTimes(1) - const page = host.history({ sessionId: SESSION, direction: 'tail' }) + const page = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(page.ok && page.page.items).toHaveLength(1) expect(page.ok && page.page.fence).toBe(1) expect(page.page.hostNow).toBe(NOW) @@ -75,11 +105,11 @@ describe('send', () => { await attach() dispatch.mockRejectedValueOnce(new Error('socket closed')) const body = hostTestMessage('add a retry') - const result = await host.send(CALLER, { - envelope: envelope('agentSession.send', { body }), - body + const params = { envelope: envelope('agentSession.send', { body }), body } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) - expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) }) it('replays a retried send from the journal without dispatching twice', async () => { @@ -87,6 +117,7 @@ describe('send', () => { const body = hostTestMessage('add a retry') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) const retry = await host.send(CALLER, params) expect(retry).toMatchObject({ ok: true, replayed: true }) expect(dispatch).toHaveBeenCalledTimes(1) @@ -98,10 +129,9 @@ describe('send', () => { const body = hostTestMessage('possibly delivered') const params = { envelope: envelope('agentSession.send', { body }), body } - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'unknown' } } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) // A thrown adapter call is indistinguishable from a lost reply, so Retry // replays the recorded outcome. @@ -110,7 +140,7 @@ describe('send', () => { value: { submission: { dispatchState: 'unknown' } } }) expect(dispatch).toHaveBeenCalledTimes(1) - const state = host.history({ sessionId: SESSION, direction: 'tail' }) + const state = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(state.ok && state.page.submissions).toHaveLength(1) }) @@ -129,6 +159,7 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({ ok: true, value: { @@ -136,7 +167,7 @@ describe('send', () => { } }) expect(dispatch).toHaveBeenCalledTimes(1) - const state = host.history({ sessionId: SESSION, direction: 'tail' }) + const state = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(state.ok && state.page.submissions).toHaveLength(1) }) @@ -151,24 +182,21 @@ describe('send', () => { const body = hostTestMessage('never written') const params = { envelope: envelope('agentSession.send', { body }), body } - await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: true, - value: { - submission: { dispatchState: 'rejected', reason: 'provider_write_failed: broken pipe' } - } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'rejected', + reason: 'provider_write_failed: broken pipe' }) // What the user's Retry does with a rejection: a fresh client message id, // which is a first delivery by construction and cannot duplicate the frame // that never left the process. - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } + const rotated = { envelope: envelope('agentSession.send', { body }), body } + await expect(host.send(CALLER, rotated)).resolves.toMatchObject({ ok: true, replayed: false }) + await expect(delivered(rotated.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'accepted' }) expect(dispatch).toHaveBeenCalledTimes(2) - const state = host.history({ sessionId: SESSION, direction: 'tail' }) + const state = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(state.ok && state.page.submissions).toHaveLength(2) }) @@ -183,10 +211,9 @@ describe('send', () => { const body = hostTestMessage('a message the provider may already hold') const params = { envelope: envelope('agentSession.send', { body }), body } - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'unknown' } } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) // No `unknown` is re-delivered under its own id, whatever its reason says, // so Retry replays the recorded outcome instead of writing again. @@ -203,6 +230,7 @@ describe('send', () => { const body = hostTestMessage('settled for good') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -223,7 +251,7 @@ describe('send', () => { expect(journal.receiptFor(params.envelope.clientOperationId)).not.toBeNull() }) - it('leaves an admitted send pending and writes no dispatch row', async () => { + it('leaves an admitted send pending once handed over', async () => { await attach() dispatch.mockImplementationOnce(async () => ({ state: 'admitted' as const })) const body = hostTestMessage('queued behind a running turn') @@ -233,9 +261,9 @@ describe('send', () => { ok: true, value: { submission: { dispatchState: 'pending', reason: null, resolvedAt: null } } }) - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + const journal = hostJournal() expect(journal.pendingSubmissions()).toHaveLength(1) }) @@ -245,6 +273,8 @@ describe('send', () => { const body = hostTestMessage('written, never acknowledged') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -281,6 +311,8 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed') + // The submission was recorded before the ledger write failed, so it is still delivered. + await delivered(params.envelope.clientOperationId) await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true, @@ -333,9 +365,8 @@ describe('send', () => { await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed') settlement.mockRestore() - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal + await delivered(params.envelope.clientOperationId) + const journal = hostJournal() await journal.rollEpoch('schema_unreadable', store.getRecord(SESSION)?.lease.runtimeFence ?? 1) expect(journal.submissions()).toHaveLength(0) @@ -362,6 +393,7 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) expect(dispatch).toHaveBeenCalledTimes(1) await store.recordOperationOutcome({ callerKey: CALLER.callerKey, @@ -393,6 +425,8 @@ describe('send', () => { const body = hostTestMessage('written, then the child died') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -411,21 +445,25 @@ describe('send', () => { it('advances an explicit retry after a ledger-unknown send is reconciled in the journal', async () => { await attach() - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal - vi.spyOn(journal, 'resolveDispatch').mockRejectedValueOnce(new Error('journal resolve failed')) + const journal = hostJournal() + const resolve = journal.resolveDispatch.bind(journal) + // The handover row lands; the provider's answer, written after the adapter took the + // message, does not. + vi.spyOn(journal, 'resolveDispatch') + .mockImplementationOnce(resolve) + .mockRejectedValueOnce(new Error('journal resolve failed')) const body = hostTestMessage('possibly delivered before persistence failed') const params = { envelope: envelope('agentSession.send', { body }), body } - await expect(host.send(CALLER, params)).rejects.toThrow('journal resolve failed') - expect(journal.submissions()).toMatchObject([ - { clientMessageId: params.envelope.clientOperationId, dispatchState: 'unknown' } - ]) + await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true }) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' + }) + // Acceptance is what the ledger answers for; delivery is the journal's to say. expect( store.listOperationRows().find((row) => row.operationId === params.envelope.clientOperationId) ?.outcome - ).toEqual({ status: 'unknown' }) + ).toMatchObject({ status: 'succeeded' }) expect(dispatch).toHaveBeenCalledTimes(1) await journal.markPendingSubmissionsUnknown(store.getRecord(SESSION)?.lease.runtimeFence ?? 1) @@ -446,7 +484,7 @@ describe('send', () => { expect(journal.submissions()).toHaveLength(1) }) - it('admits a send fenced to another generation, and replays it by id once the fence catches up', async () => { + it('admits a send fenced to another generation, delivers it once, and replays it by id', async () => { const record = await attach() const body = hostTestMessage('add a retry') const params = { @@ -457,10 +495,9 @@ describe('send', () => { ), body } - expect(await host.send(CALLER, params)).toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } + expect(await host.send(CALLER, params)).toMatchObject({ ok: true, replayed: false }) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'accepted' }) const retry = { ...params, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts index 5780359a8aec..d30af1c807a1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts @@ -125,6 +125,11 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }) }) +/** The host starting the agent with no message to deliver, as an operation that needs it does. */ +function startAgent(): Promise { + return host['serialize'](SESSION, () => host['mutationContext']().ensureAgent(SESSION)) +} + describe('settled attach retry', () => { it('settles a post-acquisition journal failure and retries without a restart', async () => { const historyFilePath = vi @@ -297,7 +302,7 @@ describe('settled attach retry', () => { ).toEqual({ status: 'pending' }) reservationUnused = true - await host.hold(SESSION, 'desktop-chat:retry') + await startAgent() expect(mintSpawnToken).toHaveBeenCalledTimes(2) expect(spawnTokens).toEqual(['spawn-1', 'spawn-2']) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -322,6 +327,8 @@ describe('settled attach retry', () => { } const first = await host.send(CALLER, unknownParams) expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // Handed over before the host dies: that is what makes the restart's answer doubt. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) await host.flushAllStreamedEvents() store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) @@ -334,8 +341,8 @@ describe('settled attach retry', () => { probeOwner: async () => ({ outcome: 'pid-absent' }), now: () => NOW }) - await host.restoreReadableSessions() - await host.hold(SESSION, 'desktop-chat:restart') + await host.restoreStartupSessions() + await startAgent() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live', handoffStage: null, @@ -351,8 +358,8 @@ describe('settled attach retry', () => { if (!sent.ok) { throw new Error(`unexpected restored send refusal: ${sent.refusal.message}`) } - expect(dispatch).toHaveBeenCalledTimes(2) - const restoredHistory = host.history({ sessionId: SESSION, direction: 'tail' }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(2)) + const restoredHistory = await host.history({ sessionId: SESSION, direction: 'tail' }) if (!restoredHistory.ok) { throw new Error(`unexpected restored history reset: ${restoredHistory.reset}`) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.test.ts index c0d29f6b9efe..4ea4b4e13a30 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.test.ts @@ -45,7 +45,7 @@ function retry(settlementId: string, deathEvidence: AgentSessionDeathEvidence) { return retryLoadedStructuredAgentSessionSettlement({ deps: { store: store() }, sessionId: SESSION, - session: { journal, fence: FENCE, acquisitionGeneration: null }, + journal, now: () => 2_000 }) } @@ -119,6 +119,31 @@ describe('pending settlement retry', () => { expect(record.lease.settlementRetryRequired).toBeUndefined() }) + it('leaves a message queued for the next child to that child', async () => { + await seedRunningTurn() + await journal.appendSubmission({ + clientMessageId: 'queued', + payloadFingerprint: 'fingerprint', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hello' }] }, + fence: FENCE, + handoverRecorded: true + }) + + await expect( + retry(`restart-eviction:${SESSION}:${FENCE}`, { + kind: 'pid-absent', + detail: 'recorded pid absent on host', + observedAt: 1_500 + }) + ).resolves.toBe(true) + + // The retry settles the generation that died; the message was never that child's. + expect(journal.submissions()).toEqual([ + expect.objectContaining({ clientMessageId: 'queued', dispatchState: 'pending' }) + ]) + expect(journal.submissions()[0]?.handedOverAt).toBeUndefined() + }) + it('writes no status row for an identity mismatch either', async () => { await seedRunningTurn() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.ts index 1000881220e9..a4e49f1d7d5f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-retry.ts @@ -1,60 +1,54 @@ -import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { attachJournal } from './structured-agent-session-attach' -import type { - StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession -} from './structured-agent-session-host-types' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import type { StructuredAgentSessionLeaseStore } from './structured-agent-session-lease-release' import { turnVerdictFromDeathEvidence } from './structured-agent-session-stale-turn-verdict' import { captureUnfinishedStructuredAgentSessionWork, settleStructuredAgentSessionDeadGeneration, - unfinishedStructuredAgentSessionWorkWasInterrupted + unfinishedStructuredAgentSessionWorkWasInterrupted, + type DeadGenerationJournal } from './structured-agent-session-dead-generation-settlement' +/** Whether a restart or an exit left this record's turn owed a settlement. The one definition: + * every trigger that retries (the attach, a start, the startup pass) asks this. */ +export function structuredAgentSessionOwesSettlement( + record: AgentSessionRecord | null +): record is AgentSessionRecord & { + lease: AgentSessionRecord['lease'] & { settlementRetryId: string } +} { + return record?.lease.settlementRetryRequired === true && !!record.lease.settlementRetryId +} + +/** Retries a durable provider-exit settlement against the conversation's own journal. The retry is + * for an earlier child, so it writes at the record's fence and leaves the conversation — and any + * message queued for the next child — alone. */ export async function retryPendingStructuredAgentSessionSettlement(input: { deps: StructuredAgentSessionHostDeps - sessions: Map sessionId: string - params: AgentSessionAttachParams + /** The conversation's journal, opened through the host's one open when it is closed. */ + openJournal: () => Promise now: () => number }): Promise { const record = input.deps.store.getRecord(input.sessionId) - if (!record?.lease.settlementRetryRequired || !record.lease.settlementRetryId) { + if (!structuredAgentSessionOwesSettlement(record)) { return true } - let journal = input.sessions.get(input.sessionId)?.journal - if (!journal) { - try { - journal = ( - await attachJournal({ - record, - params: input.params, - journalRoot: input.deps.journalRoot, - adapter: input.deps.adapter - }) - ).journal - } catch (error) { - input.deps.onEventSinkError?.({ sessionId: input.sessionId, error }) - return false - } + let journal: AgentSessionJournal | null + try { + journal = await input.openJournal() + } catch (error) { + input.deps.onEventSinkError?.({ sessionId: input.sessionId, error }) + return false } - const current = input.sessions.get(input.sessionId) - const retrySession = - current ?? - ({ - journal, - params: input.params, - fence: record.lease.runtimeFence, - hasProviderChild: false, - acquisitionGeneration: null - } as StructuredAgentSessionHostSession) - return retryLoadedStructuredAgentSessionSettlement({ - deps: input.deps, - sessionId: input.sessionId, - session: retrySession, - now: input.now - }) + return journal + ? retryLoadedStructuredAgentSessionSettlement({ + deps: input.deps, + sessionId: input.sessionId, + journal, + now: input.now + }) + : false } export async function retryLoadedStructuredAgentSessionSettlement(input: { @@ -63,23 +57,23 @@ export async function retryLoadedStructuredAgentSessionSettlement(input: { onEventSinkError?: StructuredAgentSessionHostDeps['onEventSinkError'] } sessionId: string - session: Pick + journal: DeadGenerationJournal now: () => number }): Promise { const record = input.deps.store.getRecord(input.sessionId) - if (!record?.lease.settlementRetryRequired || !record.lease.settlementRetryId) { + if (!structuredAgentSessionOwesSettlement(record)) { return true } - const retrySession = input.session - retrySession.fence = record.lease.runtimeFence + const { journal } = input + const fence = record.lease.runtimeFence const onError = (id: string, error: unknown): void => input.deps.onEventSinkError?.({ sessionId: id, error }) // Only an observed exit earns an end time; a probe-proven death never saw one. const verdict = turnVerdictFromDeathEvidence(record.lease.deathEvidence) const ok = await settleStructuredAgentSessionDeadGeneration({ - journal: retrySession.journal, + journal, sessionId: input.sessionId, - fence: retrySession.fence, + fence, settlementId: record.lease.settlementRetryId, pendingSubmissionReason: 'provider_exited_before_acknowledgement', verdict, @@ -90,8 +84,8 @@ export async function retryLoadedStructuredAgentSessionSettlement(input: { showUnexpectedExitOutcome: verdict.state === 'interrupted' && unfinishedStructuredAgentSessionWorkWasInterrupted( - captureUnfinishedStructuredAgentSessionWork(retrySession.journal), - retrySession.journal, + captureUnfinishedStructuredAgentSessionWork(journal), + journal, verdict.completedAt ), ...(record.lease.deathEvidence?.detail diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts new file mode 100644 index 000000000000..005949661012 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts @@ -0,0 +1,59 @@ +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' + +/** + * The one row a start that failed leaves in the chat, whoever saw it fail: an error row, so the + * reason outlives any error strip. Keyed by the start — the child's generation, or the oldest + * message it was for when no child was ever published — so a second report of the same failure + * revises the row instead of adding one. + */ +export function structuredAgentSessionStartFailureRow( + startKey: string, + text: string +): JournalLifecycleMutationInput { + return { + kind: 'item', + identity: { provider: 'orca', clientMessageId: `start-failure:${startKey}` }, + body: { kind: 'status', text: boundJournalStatusText(text), tone: 'error' } + } +} + +/** + * A start the delivery loop needed and did not get: the start's row, and every queued message + * rejected with the same words. Writes nothing when nothing is still queued: a start whose + * messages Stop withdrew did not fail anyone. + */ +export async function recordStructuredAgentSessionStartFailure( + session: Pick & { fence: number }, + failure: { startKey: string | null; text: string } +): Promise { + const oldest = oldestQueuedSubmission(session) + if (!oldest) { + return + } + const startKey = failure.startKey ?? oldest.clientMessageId + await session.journal.appendLifecycleBatch({ + settlementId: `start-failure:${startKey}`, + fence: session.fence, + recovered: true, + mutations: [structuredAgentSessionStartFailureRow(startKey, failure.text)] + }) + await session.journal.rejectQueuedSubmissions(session.fence, failure.text) +} + +export function oldestQueuedSubmission( + session: Pick +): ReturnType[number] | undefined { + let oldest: ReturnType + for (const submission of session.journal.submissions()) { + if ( + isQueuedAgentJournalSubmission(submission) && + (oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0)) + ) { + oldest = submission + } + } + return oldest +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index 0fd544ec9eea..3fd3c24a1001 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -19,10 +19,9 @@ function startedSession(): StructuredAgentSessionUnexpectedExitSession & { journal: { appendLifecycleBatch: ReturnType } } { return { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), // Nothing ran: the start failed before any response or acknowledged prompt. snapshot: () => ({ items: [] }), appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), @@ -56,7 +55,6 @@ function contextFor(session: StructuredAgentSessionUnexpectedExitSession) { sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), publishFence: vi.fn(), - hasResumeCapableHolder: () => true, serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1 } @@ -73,20 +71,21 @@ const ended = { } describe('a provider that ends before it finished starting', () => { - it('tells the user why, even with no response in progress, and does not auto-resume', async () => { + it('tells the user why, even with no response in progress', async () => { const session = startedSession() - const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), { + await settleUnexpectedStructuredAgentSessionExit(contextFor(session), { ...ended, startupUnproven: true }) - expect(ticket).toBeNull() expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ mutations: [ expect.objectContaining({ - body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + // The same row the delivery loop writes for a failed start: an error, keyed by it. + identity: { provider: 'orca', clientMessageId: `start-failure:${GENERATION}` }, + body: { kind: 'status', text: providerStartupFailureOutcome(REASON), tone: 'error' } }) ] }) @@ -94,26 +93,30 @@ describe('a provider that ends before it finished starting', () => { expect(providerStartupFailureOutcome(REASON)).toContain('not signed in') }) - it('keeps an ordinary idle exit silent and resumable', async () => { + it('keeps an ordinary idle exit silent', async () => { const session = startedSession() - const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) + await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) - expect(ticket).not.toBeNull() + expect(session.child).toBeNull() expect(session.journal.appendLifecycleBatch).not.toHaveBeenCalled() }) it("reads a start that failed off the host's own phase when the provider omits the flag", async () => { - const session = { ...startedSession(), providerChildPhase: 'starting' as const } + const session = { + ...startedSession(), + child: { generation: GENERATION, fence: 7, phase: 'starting' as const } + } - const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) + await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) - expect(ticket).toBeNull() expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ mutations: [ expect.objectContaining({ - body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + // The same row the delivery loop writes for a failed start: an error, keyed by it. + identity: { provider: 'orca', clientMessageId: `start-failure:${GENERATION}` }, + body: { kind: 'status', text: providerStartupFailureOutcome(REASON), tone: 'error' } }) ] }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-pass-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-pass-test-rig.ts new file mode 100644 index 000000000000..fa5fdf8adc6b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-pass-test-rig.ts @@ -0,0 +1,261 @@ +// Real hosts over one real store, journals and saved-status file, booted in turn the way app runs +// follow each other, for the tests of what a restarted host owes each chat before anyone opens it. +// Every journal open calls the adapter's `historyFilePath` once, so that is the open counter, and +// holding it holds the open. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionStatusEvent } from '../../../shared/agent-session-wire' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { AgentSessionSavedStatusStore } from '../../runtime/agent-session-saved-status-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import type { AgentSessionAttachParams } from './structured-agent-session-attach' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { + HOST_TEST_LOCATION, + HOST_TEST_NOW, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +export const STARTUP_CALLER = { callerKey: 'client-1' } + +export type StartupRig = { + root: string + store: AgentSessionRecordStore + host: StructuredAgentSessionHost + savedStatus: AgentSessionSavedStatusStore + acquire: Mock + dispatch: Mock + /** Called once per journal open, with the session id; replace its implementation to hold one. */ + historyFilePath: Mock<(sessionId: string) => Promise> + probeOwner: Mock<(record: AgentSessionRecord) => Promise> + statusEvents: AgentSessionStatusEvent[] + sink: { publish: Mock; forget: Mock } + /** Creates the chat (or attaches it again at its fence), lists its tab unless told not to, and + * sends `message` when one is given, waiting for its dispatch. */ + chat: ( + sessionId: string, + options?: { workspaceId?: string; listed?: boolean; message?: string } + ) => Promise + opensOf: (sessionId: string) => number + /** A clean quit: teardown settles and saves, then the saved-status file closes. */ + quit: () => Promise + /** A crash: nothing settles or flushes; the process's timers and handles simply stop. */ + crash: () => Promise + /** A new app run over the same files; the counters and status stream start empty. */ + boot: (deps?: Partial) => Promise + dispose: () => Promise +} + +export function startupAttachParams( + sessionId: string, + workspaceId: string, + fence: number | null +): AgentSessionAttachParams { + return hostTestAttachParams(fence, { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: '' + }, + location: { ...HOST_TEST_LOCATION, workspaceId }, + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + }) +} + +export async function createStartupRig(): Promise { + resetHostTestOperationIds() + const root = await mkdtemp(join(tmpdir(), 'orca-startup-pass-')) + const directory = join(root, 'store') + const statusEvents: AgentSessionStatusEvent[] = [] + const sink = { publish: vi.fn(), forget: vi.fn() } + const historyFilePath = vi.fn(async (_sessionId: string): Promise => null) + const probeOwner: StartupRig['probeOwner'] = vi.fn(async () => ({ outcome: 'pid-absent' })) + let ordinal = 0 + const acquire: StartupRig['acquire'] = vi.fn(async ({ fence, spawnToken, identity }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${identity.sessionId}-${fence}`, + handle: + identity.providerHandle.kind === 'codex' + ? { provider: 'codex' as const, threadId: identity.providerHandle.threadId } + : { provider: 'codex' as const, threadId: `thread-${identity.sessionId}` }, + origin: rig.store.getRecord(identity.sessionId)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: HOST_TEST_NOW + } + })) + const dispatch: StartupRig['dispatch'] = vi.fn(async (input) => { + ordinal += 1 + return { + state: 'accepted', + providerIdentity: { + provider: 'codex', + threadId: `thread-${input.sessionId}`, + turnId: `turn-${ordinal}`, + ordinal + } + } + }) + const openAll: { store: AgentSessionRecordStore; savedStatus: AgentSessionSavedStatusStore }[] = + [] + const makeHost = ( + store: AgentSessionRecordStore, + savedStatus: AgentSessionSavedStatusStore, + deps: Partial + ) => + new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: async () => ({ cancelled: true }), + answerPrompt: async ({ commit }) => commit(), + setOption: async () => undefined, + historyFilePath: ({ identity }) => historyFilePath(identity.sessionId) + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + probeOwner, + now: () => HOST_TEST_NOW, + statusSink: sink, + savedStatus, + ...deps + }) + const open = async (deps: Partial) => { + const store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) + const savedStatus = AgentSessionSavedStatusStore.open(root) + openAll.push({ store, savedStatus }) + const host = makeHost(store, savedStatus, deps) + host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) + return { store, savedStatus, host } + } + const first = await open({}) + const rig: StartupRig = { + root, + ...first, + acquire, + dispatch, + historyFilePath, + probeOwner, + statusEvents, + sink, + chat: async (sessionId, options = {}) => { + const workspaceId = options.workspaceId ?? 'workspace-1' + const attach = () => + rig.host.attach( + STARTUP_CALLER, + startupAttachParams( + sessionId, + workspaceId, + rig.store.getRecord(sessionId)?.lease.runtimeFence ?? null + ) + ) + let attached = await attach() + if (!attached.ok && attached.refusal.code === 'agent_session_checkpoint_stale') { + attached = await attach() + } + if (!attached.ok) { + throw new Error(`attach refused: ${attached.refusal.code} ${attached.refusal.message}`) + } + if (options.listed !== false) { + await rig.store.setSessionTabVisibility(sessionId, true) + } + if (options.message === undefined) { + return + } + const dispatched = dispatch.mock.calls.length + const body = hostTestMessage(options.message) + const sent = await rig.host.send(STARTUP_CALLER, { + body, + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: attached.fence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId, + fields: { body } + }) + } + }) + if (!sent.ok) { + throw new Error(`send refused: ${sent.refusal.code}`) + } + await vi.waitFor(() => expect(dispatch.mock.calls.length).toBeGreaterThan(dispatched)) + }, + opensOf: (sessionId) => historyFilePath.mock.calls.filter(([id]) => id === sessionId).length, + quit: async () => { + await rig.host.flushAllStreamedEvents() + rig.savedStatus.close() + }, + crash: async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host's private collaborators, reached only to stop them as a dead process would. + const internals = rig.host as unknown as { + runtimeState: { stopLeaseRenewal: () => void } + lifetime: { dispose: () => void } + sessions: Map Promise } }> + } + internals.runtimeState.stopLeaseRenewal() + internals.lifetime.dispose() + for (const session of internals.sessions.values()) { + await session.journal.close() + } + internals.sessions.clear() + }, + boot: async (deps = {}) => { + // What the new run saw, and nothing the previous one did. + historyFilePath.mockClear() + sink.publish.mockClear() + sink.forget.mockClear() + statusEvents.length = 0 + const next = await open(deps) + rig.store = next.store + rig.savedStatus = next.savedStatus + rig.host = next.host + return next.host + }, + dispose: async () => { + await rig.host.flushAllStreamedEvents().catch(() => undefined) + openAll.forEach(({ savedStatus }) => savedStatus.close()) + await rm(root, { recursive: true, force: true }) + } + } + return rig +} + +/** The newest row the status stream carried for a session. */ +export function latestStatus(rig: StartupRig, sessionId: string) { + for (const event of rig.statusEvents.toReversed()) { + if (event.type === 'status' && event.session.sessionId === sessionId) { + return event.session + } + if (event.type === 'snapshot') { + const found = event.sessions.find((session) => session.sessionId === sessionId) + if (found) { + return found + } + } + } + return undefined +} + +/** Whether the host holds this conversation open right now. */ +export function isOpen(rig: StartupRig, sessionId: string): boolean { + return rig.host.hasSession(sessionId) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-pass.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-pass.test.ts new file mode 100644 index 000000000000..8a3a9ecef411 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-pass.test.ts @@ -0,0 +1,387 @@ +// What a restarted host owes each listed chat before anyone opens it: a status row from its saved +// copy when the journal still stands where that copy was taken, and otherwise one open, settled and +// closed again, off every reader's path. + +import { existsSync } from 'node:fs' +import { rm, writeFile } from 'node:fs/promises' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, + type StructuredAgentSessionSavedStatus +} from '../../../shared/structured-agent-session-saved-status' +import { AgentSessionSavedStatusStore } from '../../runtime/agent-session-saved-status-store' +import { journalDatabaseFile, journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { HOST_TEST_NOW } from './structured-agent-session-host-test-data' +import { + createStartupRig, + isOpen, + latestStatus, + type StartupRig +} from './structured-agent-session-startup-pass-test-rig' + +let rig: StartupRig + +type HostLock = { serialize: (id: string, task: () => Promise) => Promise } + +beforeEach(async () => { + rig = await createStartupRig() +}) + +afterEach(async () => { + await rig.dispose() +}) + +/** Every row the current run published for a session, in order. */ +function rowsOf(sessionId: string) { + return rig.statusEvents.flatMap((event) => + event.type === 'status' && event.session.sessionId === sessionId + ? [event.session] + : event.type === 'snapshot' + ? event.sessions.filter((session) => session.sessionId === sessionId) + : [] + ) +} + +async function turnStates(sessionId: string): Promise { + const { items } = await rig.host.journalSnapshot(sessionId) + return items.flatMap((item) => (item.body.kind === 'turn' ? [item.body.state] : [])) +} + +function journalDir(sessionId: string): string { + return journalDirectoryFor(rig.root, { workspaceId: 'workspace-1', sessionId }) +} + +/** A second run's send, then a crash with the turn still running and its child alive. */ +async function crashMidTurn(sessionId: string, text: string): Promise { + const recorded: StructuredAgentSessionSavedStatus[] = [] + const host = await rig.boot({ + savedStatus: { + read: (id) => rig.savedStatus.read(id), + prune: (keep) => rig.savedStatus.prune(keep), + record: (id, saved) => { + recorded.push(saved) + rig.savedStatus.record(id, saved) + } + } + }) + await host.restoreStartupSessions() + await rig.chat(sessionId, { message: text }) + await rig.host + .collaboratorsForTests() + .sessions.get(sessionId)! + .journal.appendItem( + { provider: 'codex', threadId: `thread-${sessionId}`, turnId: 'turn-running', ordinal: 99 }, + { kind: 'turn', turnId: 'turn-running', state: 'running', startedAt: 1 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence } + ) + // Every publish of the running turn has run, and the flush timer fired: whatever it left in the + // saved copy is on disk. + await new Promise((resolve) => setTimeout(resolve, 0)) + rig.savedStatus.flush() + await rig.crash() + // Only a settled chat is ever saved: a running turn writes nothing. + expect(recorded.filter((saved) => saved.projection.status === 'working')).toEqual([]) +} + +describe('listing from a saved status', () => { + it('lists a settled chat from its saved status without opening its journal (C1)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(0) + expect(isOpen(rig, 'session-a')).toBe(false) + expect(latestStatus(rig, 'session-a')).toMatchObject({ + sessionId: 'session-a', + workspaceId: 'workspace-1', + status: 'idle', + latestPrompt: 'first question' + }) + }) + + it('publishes saved rows for the active workspace first, then the visible ones', async () => { + await rig.chat('session-r', { workspaceId: 'ws-rest', message: 'rest' }) + await rig.chat('session-v', { workspaceId: 'ws-visible', message: 'visible' }) + await rig.chat('session-x', { workspaceId: 'ws-active', message: 'active' }) + await rig.quit() + const host = await rig.boot() + + await host.restoreStartupSessions({ + activeWorkspaceId: 'ws-active', + visibleWorkspaceIds: new Set(['ws-visible']) + }) + + expect(['session-r', 'session-v', 'session-x'].map((id) => rig.opensOf(id))).toEqual([0, 0, 0]) + const firstRows = rig.statusEvents.flatMap((event) => + event.type === 'status' ? [event.session.sessionId] : [] + ) + expect([...new Set(firstRows)]).toEqual(['session-x', 'session-v', 'session-r']) + }) + + it('opens a chat whose saved copy an older projection version wrote (C3)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const stale = AgentSessionSavedStatusStore.open(rig.root) + const saved = stale.read('session-a') + expect(saved?.v).toBe(STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION) + stale.record('session-a', { ...saved!, v: saved!.v - 1 }) + stale.close() + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(1) + }) + + it('shows a model switched while idle, read from the record rather than the saved copy (C4)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const host = await rig.boot() + const record = rig.store.getRecord('session-a') + await rig.store.replaceSessionOptions({ + sessionId: 'session-a', + fence: record?.lease.runtimeFence ?? 0, + options: { model: 'gpt-switched' }, + now: HOST_TEST_NOW + }) + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(0) + expect(latestStatus(rig, 'session-a')).toMatchObject({ model: 'gpt-switched' }) + }) + + it('serves a chat that never had a turn from its saved copy, as its open would (C5)', async () => { + await rig.chat('session-a') + await rig.quit() + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(0) + expect(latestStatus(rig, 'session-a')).toMatchObject({ status: null, latestPrompt: '' }) + }) + + it('publishes the saved row as a replay: no rename edge, no start, no completion (C6)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const onSessionStatusChanged = vi.fn() + const completions = vi.fn() + const host = await rig.boot({ onSessionStatusChanged }) + host.subscribeTurnCompletions({ id: 'completions', emit: completions }) + + await host.restoreStartupSessions() + + const seen = onSessionStatusChanged.mock.calls.filter( + ([summary]) => summary.sessionId === 'session-a' + ) + expect(seen).toHaveLength(1) + expect(seen[0]?.[1]).toEqual({ replay: true }) + expect(seen[0]?.[0].hostExecutionPhase).toBeUndefined() + expect(completions).not.toHaveBeenCalled() + }) + + it('never lists a crashed turn from the settled copy before it (C2)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + await crashMidTurn('session-a', 'second question') + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(1) + expect(latestStatus(rig, 'session-a')).toMatchObject({ + status: 'idle', + latestPrompt: 'second question' + }) + // The settled turn before it is never shown. + expect(rowsOf('session-a').filter((row) => row.latestPrompt !== 'second question')).toEqual([]) + expect(rig.store.getRecord('session-a')?.lease.settlementRetryRequired).toBeUndefined() + // Settled as `unverifiable`, so no verdict; the dead-generation crash verdict is a follow-up. + expect(latestStatus(rig, 'session-a')?.turnOutcome).toBeUndefined() + expect(await turnStates('session-a')).toEqual(['unverifiable']) + }) + + it.each([ + ['saved', true], + ['lost', false] + ])( + 'lists what a quit mid-turn settled, never the finished turn before it, its save %s (C10)', + async (_case, saves) => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + // A run whose saved-status writes never land: its entry stays at the first turn's position. + const dropped = { + read: (id: string) => rig.savedStatus.read(id), + prune: (keep: (id: string) => boolean) => rig.savedStatus.prune(keep), + record: () => {} + } + const second = await rig.boot(saves ? {} : { savedStatus: dropped }) + await second.restoreStartupSessions() + await rig.chat('session-a', { message: 'second question' }) + await rig.quit() + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(latestStatus(rig, 'session-a')).toMatchObject({ latestPrompt: 'second question' }) + expect(rowsOf('session-a').filter((row) => row.latestPrompt === 'first question')).toEqual([]) + } + ) + + it('drops saved copies of chats that no longer exist (C11)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const orphan = AgentSessionSavedStatusStore.open(rig.root) + orphan.record('session-gone', { ...orphan.read('session-a')! }) + orphan.close() + const host = await rig.boot() + + await host.restoreStartupSessions() + rig.savedStatus.flush() + + expect(rig.savedStatus.read('session-gone')).toBeNull() + expect(rig.savedStatus.read('session-a')).not.toBeNull() + }) + + it('keeps status and reads working when the saved-status store throws (C12)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const broken = { + read: () => { + throw new Error('saved status unreadable') + }, + record: () => { + throw new Error('saved status unwritable') + }, + prune: () => { + throw new Error('saved status unprunable') + } + } + const host = await rig.boot({ savedStatus: broken }) + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(1) + expect(latestStatus(rig, 'session-a')).toMatchObject({ latestPrompt: 'first question' }) + const page = await host.history({ sessionId: 'session-a', direction: 'tail' }) + expect(page.ok).toBe(true) + }) +}) + +describe('a saved row is what the open would publish (C9b)', () => { + it('equals the row a real first open publishes for a settled chat', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const seededHost = await rig.boot() + await seededHost.restoreStartupSessions() + const seeded = latestStatus(rig, 'session-a') + expect(rig.opensOf('session-a')).toBe(0) + await rig.quit() + const openedHost = await rig.boot({ savedStatus: undefined }) + + await openedHost.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(1) + expect(latestStatus(rig, 'session-a')).toEqual(seeded) + }) + + it('opens a chat whose last run left a compaction prepared, and settles it', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + const host = await rig.boot() + const fence = rig.store.getRecord('session-a')?.lease.runtimeFence ?? 0 + await rig.store.setConversationCommand('session-a', fence, { + command: 'compact', + state: 'unknown', + phase: 'prepared', + runtimeFence: fence, + operationId: 'compact-op-1', + callerKey: 'client-1' + }) + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(1) + expect(rig.store.getRecord('session-a')?.conversationCommand?.phase).toBe('committed') + }) +}) + +describe('what the pass opens', () => { + it('keeps a chat with an unusable journal listed, opens it once, and refuses its read (L2, C8)', async () => { + await rig.chat('session-a', { message: 'first question' }) + await rig.quit() + await writeFile(journalDatabaseFile(journalDir('session-a')), Buffer.alloc(8192, 7)) + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(rig.opensOf('session-a')).toBe(1) + expect(latestStatus(rig, 'session-a')).toBeUndefined() + expect(host.listPersistedSessionTabs(rig.store.listVisibleSessionIds())).toEqual([ + { sessionId: 'session-a', workspaceId: 'workspace-1', agent: 'codex' } + ]) + await expect(host.history({ sessionId: 'session-a', direction: 'tail' })).rejects.toThrow( + /^agent_session_journal_unreadable$/ + ) + }) + + it('founds no journal for a listed chat that has none, and it still reads (L3)', async () => { + await rig.chat('session-a') + await rig.quit() + await rm(journalDir('session-a'), { recursive: true, force: true }) + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(existsSync(journalDatabaseFile(journalDir('session-a')))).toBe(false) + expect(rig.opensOf('session-a')).toBe(0) + const page = await host.history({ sessionId: 'session-a', direction: 'tail' }) + expect(page.ok && page.page.items).toEqual([]) + }) + + it('neither lists nor opens a chat whose tab closes before its step (L7)', async () => { + await rig.chat('session-a', { message: 'first' }) + await rig.chat('session-b', { message: 'second' }) + await rig.quit() + const host = await rig.boot() + // Queued on the chat's own lock ahead of the pass, as a close landing mid-pass would be. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host's private per-session lock, reached to order a close ahead of the pass. + const { serialize } = host as unknown as HostLock + const closed = serialize('session-b', () => + rig.store.setSessionTabVisibility('session-b', false) + ) + + await host.restoreStartupSessions() + await closed + + expect(latestStatus(rig, 'session-a')).toMatchObject({ latestPrompt: 'first' }) + expect(latestStatus(rig, 'session-b')).toBeUndefined() + expect(rig.opensOf('session-b')).toBe(0) + }) + + it('opens no chat whose tab closed between the status check and its open (L7)', async () => { + await rig.chat('session-a', { message: 'first' }) + await rig.chat('session-b', { message: 'second' }) + await rig.crash() + const reconcile = Promise.withResolvers() + const probing = Promise.withResolvers() + rig.probeOwner.mockImplementation(async () => { + probing.resolve() + await reconcile.promise + return { outcome: 'pid-absent' } + }) + const host = await rig.boot() + + const pass = host.restoreStartupSessions() + await probing.promise + await rig.store.setSessionTabVisibility('session-b', false) + reconcile.resolve() + await pass + + expect(rig.opensOf('session-a')).toBe(1) + expect(rig.opensOf('session-b')).toBe(0) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-retry.test.ts new file mode 100644 index 000000000000..1718747a6f4f --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-retry.test.ts @@ -0,0 +1,139 @@ +// A startup reconcile that fails leaves a crashed chat's turn unsettled: only reconcile can say its +// agent is gone. The first clean reconcile after it, from a bounded retry or from anyone, must +// settle every listed chat that owes it, not only the ones some reader happened to open. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { + createStartupRig, + isOpen, + latestStatus, + type StartupRig +} from './structured-agent-session-startup-pass-test-rig' + +let rig: StartupRig + +beforeEach(async () => { + rig = await createStartupRig() +}) + +afterEach(async () => { + await rig.dispose() +}) + +// The settlement a clean reconcile starts is not awaited by it; allow for a loaded runner. +const SETTLED_WITHIN = { timeout: 10_000 } + +const owesSettlement = (sessionId: string) => + rig.store.getRecord(sessionId)?.lease.settlementRetryRequired === true + +/** A second run's send, then a crash with the provider's turn still running. */ +async function crashMidTurn(sessionId: string): Promise { + await rig.chat(sessionId, { message: 'first' }) + await rig.quit() + const host = await rig.boot() + await host.restoreStartupSessions() + await rig.chat(sessionId, { message: 'second' }) + const { journal } = rig.host.collaboratorsForTests().sessions.get(sessionId)! + // The provider's turn is still running when the app dies. + await journal.appendItem( + { provider: 'codex', threadId: `thread-${sessionId}`, turnId: 'turn-running', ordinal: 99 }, + { kind: 'turn', turnId: 'turn-running', state: 'running', startedAt: 1 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence } + ) + await rig.crash() +} + +/** Settled as `unverifiable`, so no verdict; the dead-generation crash verdict is a follow-up. */ +function expectSettledUnverifiable(snapshot: { items: readonly AgentJournalRenderItem[] }): void { + expect(latestStatus(rig, 'session-a')?.turnOutcome).toBeUndefined() + expect( + snapshot.items.flatMap((item) => (item.body.kind === 'turn' ? [item.body.state] : [])) + ).toEqual(['unverifiable']) +} + +/** Boots with every owner probe failing until `probe.fail` is cleared. */ +async function bootWithFailingReconcile(deps: Partial = {}) { + const probe = { fail: true } + rig.probeOwner.mockImplementation(async () => { + if (probe.fail) { + throw new Error('probe unavailable') + } + return { outcome: 'pid-absent' } + }) + const host = await rig.boot({ savedStatus: undefined, ...deps }) + return { host, probe } +} + +describe('a startup reconcile that failed', () => { + it('settles a crashed chat nobody opened once a later reconcile succeeds', async () => { + await crashMidTurn('session-a') + const { host, probe } = await bootWithFailingReconcile({ + startupReconcileRetryDelaysMs: [60_000] + }) + await host.restoreStartupSessions() + // The pass opened it for status and closed it again; its turn is still read as running. + expect(isOpen(rig, 'session-a')).toBe(false) + expect(latestStatus(rig, 'session-a')?.status).toBe('working') + + probe.fail = false + await host.reconcileRestartLeases() + + // The settled row lands before the settlement's record write clears the flag. + await vi.waitFor(() => { + expect(latestStatus(rig, 'session-a')?.status).toBe('idle') + expect(owesSettlement('session-a')).toBe(false) + }, SETTLED_WITHIN) + expectSettledUnverifiable(await rig.host.journalSnapshot('session-a')) + }) + + it('retries the reconcile itself, so no attach is needed', async () => { + await crashMidTurn('session-a') + const { host, probe } = await bootWithFailingReconcile({ + startupReconcileRetryDelaysMs: [50, 50, 50] + }) + await host.restoreStartupSessions() + expect(latestStatus(rig, 'session-a')?.status).toBe('working') + const acquired = rig.acquire.mock.calls.length + + probe.fail = false + + // The settled row lands before the settlement's record write clears the flag. + await vi.waitFor(() => { + expect(latestStatus(rig, 'session-a')?.status).toBe('idle') + expect(owesSettlement('session-a')).toBe(false) + }, SETTLED_WITHIN) + expectSettledUnverifiable(await rig.host.journalSnapshot('session-a')) + expect(rig.acquire.mock.calls.length).toBe(acquired) + }) + + it('gives up after its retries, leaving the chat to its own attach', async () => { + await crashMidTurn('session-a') + const onEventSinkError = vi.fn() + const { host } = await bootWithFailingReconcile({ + startupReconcileRetryDelaysMs: [20, 20, 20], + onEventSinkError + }) + await host.restoreStartupSessions() + await vi.waitFor( + () => + expect(onEventSinkError).toHaveBeenCalledWith({ + sessionId: 'startup-pass', + error: expect.objectContaining({ message: expect.stringContaining('after its retries') }) + }), + SETTLED_WITHIN + ) + expect(latestStatus(rig, 'session-a')?.status).toBe('working') + }) + + it('stops retrying once quit begins', async () => { + await crashMidTurn('session-a') + const { host } = await bootWithFailingReconcile({ startupReconcileRetryDelaysMs: [50, 50, 50] }) + await host.restoreStartupSessions() + await rig.crash() + const probes = rig.probeOwner.mock.calls.length + await new Promise((resolve) => setTimeout(resolve, 250)) + expect(rig.probeOwner.mock.calls.length).toBe(probes) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.test.ts new file mode 100644 index 000000000000..fcb2a7658134 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.test.ts @@ -0,0 +1,245 @@ +// The startup pass's second phase: after reconcile, what a restart left owed is settled for every +// listed chat and every open conversation, one chat at a time, without holding any reader behind +// more than the chat in progress, and without leaving open what only the pass wanted open. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION } from '../../../shared/structured-agent-session-saved-status' +import { probeJournalCursor } from '../agent-session-journal/journal-cursor-probe' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { + createStartupRig, + isOpen, + type StartupRig +} from './structured-agent-session-startup-pass-test-rig' + +let rig: StartupRig + +beforeEach(async () => { + rig = await createStartupRig() +}) + +afterEach(async () => { + await rig.dispose() +}) + +/** Holds every journal open while `holding`, releasable one session at a time. */ +function gateOpens() { + const held = new Map void>() + const gate = { holding: true, held, release: (sessionId: string) => held.get(sessionId)?.() } + rig.historyFilePath.mockImplementation(async (sessionId) => { + if (gate.holding) { + const opened = Promise.withResolvers() + held.set(sessionId, opened.resolve) + await opened.promise + } + return null + }) + return gate +} + +const owesSettlement = (sessionId: string) => + rig.store.getRecord(sessionId)?.lease.settlementRetryRequired === true + +describe('settling what a restart left owed', () => { + it('reads a restart-cut turn before reconcile, and settles it once reconcile lands (L6)', async () => { + // Hidden: no tab, so only the pass's settlement of open conversations can reach it. + await rig.chat('session-h', { listed: false, message: 'cut off' }) + await rig.crash() + const reconcile = Promise.withResolvers() + const probing = Promise.withResolvers() + rig.probeOwner.mockImplementation(async () => { + probing.resolve() + await reconcile.promise + return { outcome: 'pid-absent' } + }) + const host = await rig.boot() + const acquired = rig.acquire.mock.calls.length + + const pass = host.restoreStartupSessions() + await probing.promise + // A read waits on no reconcile. + const page = await host.history({ sessionId: 'session-h', direction: 'tail' }) + expect(page.ok).toBe(true) + reconcile.resolve() + await pass + + expect(owesSettlement('session-h')).toBe(false) + expect(rig.store.getRecord('session-h')?.lease.settlementRetryId).toBeUndefined() + expect(rig.acquire.mock.calls.length).toBe(acquired) + }) + + it('settles a listed chat whose child outlived a crash even when its saved status matched (L18)', async () => { + await rig.chat('session-a', { message: 'one' }) + await rig.chat('session-b', { message: 'two' }) + await rig.quit() + const second = await rig.boot() + await second.restoreStartupSessions() + await rig.chat('session-a') + rig.savedStatus.flush() + await rig.crash() + const dir = journalDirectoryFor(rig.root, { + workspaceId: 'workspace-1', + sessionId: 'session-a' + }) + const saved = rig.savedStatus.read('session-a') + expect(saved?.v).toBe(STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION) + expect(saved?.cursor).toEqual(probeJournalCursor(dir, 'session-a')) + const host = await rig.boot() + + await host.restoreStartupSessions() + + expect(owesSettlement('session-a')).toBe(false) + expect(rig.opensOf('session-a')).toBe(1) + expect(rig.opensOf('session-b')).toBe(0) + // Settling an idle chat changes nothing it shows: the seeded row stands (C9b). + const rows = rig.statusEvents.flatMap((event) => + event.type === 'status' && event.session.sessionId === 'session-a' ? [event.session] : [] + ) + expect(rows.length).toBeGreaterThan(0) + rows.forEach((row) => expect(row).toEqual(rows[0])) + }) + + it('settles open conversations again after a later reconcile succeeds (L19)', async () => { + await rig.chat('session-a', { message: 'cut off' }) + await rig.crash() + let reconcileFails = true + rig.probeOwner.mockImplementation(async () => { + if (reconcileFails) { + throw new Error('probe unavailable') + } + return { outcome: 'pid-absent' } + }) + const host = await rig.boot() + await host.restoreStartupSessions() + await host.history({ sessionId: 'session-a', direction: 'tail' }) + expect(rig.store.getRecord('session-a')?.lease.unreconciled).toBe(true) + + reconcileFails = false + await host.reconcileRestartLeases() + + // The settlement a clean reconcile starts is not awaited by it; allow for a loaded runner. + await vi.waitFor(() => expect(owesSettlement('session-a')).toBe(false), { timeout: 10_000 }) + }) +}) + +describe('one chat at a time, readers first', () => { + it('opens by tier, lets a read of another chat through, and settles a chat a read opened (L17)', async () => { + await rig.chat('session-r', { workspaceId: 'ws-rest', message: 'rest' }) + await rig.chat('session-v', { workspaceId: 'ws-visible', message: 'visible' }) + await rig.chat('session-x', { workspaceId: 'ws-active', message: 'active' }) + await rig.crash() + const host = await rig.boot() + const gate = gateOpens() + + const pass = host.restoreStartupSessions({ + activeWorkspaceId: 'ws-active', + visibleWorkspaceIds: new Set(['ws-visible']) + }) + await vi.waitFor(() => expect([...gate.held.keys()]).toEqual(['session-x'])) + gate.holding = false + // The pass is inside session-x's open; a read of a later chat does not wait for it. + const page = await host.history({ sessionId: 'session-r', direction: 'tail' }) + expect(page.ok).toBe(true) + expect(owesSettlement('session-r')).toBe(true) + gate.release('session-x') + await pass + + expect(rig.historyFilePath.mock.calls.map(([sessionId]) => sessionId)).toEqual([ + 'session-x', + 'session-r', + 'session-v' + ]) + expect(owesSettlement('session-r')).toBe(false) + expect(owesSettlement('session-v')).toBe(false) + expect(owesSettlement('session-x')).toBe(false) + }) + + it('opens a chat once when a read queues behind or ahead of the pass (L5, L20)', async () => { + await rig.chat('session-x', { message: 'first' }) + await rig.chat('session-y', { message: 'second' }) + await rig.quit() + const host = await rig.boot({ savedStatus: undefined }) + const gate = gateOpens() + + const pass = host.restoreStartupSessions() + await vi.waitFor(() => expect([...gate.held.keys()]).toEqual(['session-x'])) + const reads = [ + host.history({ sessionId: 'session-y', direction: 'tail' }), + host.history({ sessionId: 'session-y', direction: 'tail' }) + ] + await vi.waitFor(() => expect(gate.held.has('session-y')).toBe(true)) + gate.release('session-x') + gate.holding = false + gate.release('session-y') + await Promise.all([pass, ...reads]) + + expect(rig.opensOf('session-y')).toBe(1) + expect(isOpen(rig, 'session-y')).toBe(true) + }) + + it('leaves no journal open that only it opened, on a first boot where every chat misses (L21)', async () => { + const passOnly = ['session-p1', 'session-p2', 'session-p3', 'session-p4'] + for (const sessionId of passOnly) { + await rig.chat(sessionId, { message: sessionId }) + } + await rig.chat('session-u', { message: 'user first' }) + await rig.quit() + const host = await rig.boot({ savedStatus: undefined }) + await host.history({ sessionId: 'session-u', direction: 'tail' }) + + await host.restoreStartupSessions() + + for (const sessionId of passOnly) { + expect(rig.opensOf(sessionId)).toBe(1) + expect(isOpen(rig, sessionId)).toBe(false) + expect(rig.sink.publish).toHaveBeenCalledWith( + expect.objectContaining({ sessionId, latestPrompt: sessionId }), + expect.anything() + ) + } + // Opened by a reader before its step, so not the pass's to close. + expect(isOpen(rig, 'session-u')).toBe(true) + expect(rig.sink.forget).not.toHaveBeenCalled() + }) +}) + +describe('recovery at boot', () => { + it('stops a surviving owner of every resolvable record, hidden or listed, and no other (L8)', async () => { + await rig.chat('session-r', { message: 'released' }) + await rig.quit() + await rig.boot() + const pids: Record = { 'session-h': 5001, 'session-l': 5002, 'session-c': 5003 } + const acquire = rig.acquire.getMockImplementation()! + rig.acquire.mockImplementation(async (input) => { + const acquired = await acquire(input) + const pid = pids[input.identity.sessionId] ?? acquired.process.pid + return { ...acquired, process: { ...acquired.process, pid } } + }) + await rig.chat('session-h', { listed: false }) + await rig.chat('session-l') + await rig.chat('session-c') + await rig.store.transitionHandoff('session-c', (record) => ({ + ...record, + lease: { ...record.lease, claimStatus: 'conflicted', handoffStage: 'manual-recovery' } + })) + await rig.crash() + const alive = new Set(Object.values(pids)) + rig.probeOwner.mockImplementation(async (record) => + alive.has(record.lease.ownerProcess?.pid ?? -1) + ? { outcome: 'identity-matched', matchedOn: ['process-start-time'] } + : { outcome: 'pid-absent' } + ) + const stopOwnerProcess = vi.fn((pid: number) => { + alive.delete(pid) + }) + const host = await rig.boot({ stopOwnerProcess }) + + await host.restoreStartupSessions() + + const stopped = new Set(stopOwnerProcess.mock.calls.map(([pid]) => pid)) + expect(stopped).toEqual(new Set([5001, 5002])) + expect(rig.store.getRecord('session-h')?.lease).toMatchObject({ claimStatus: 'released' }) + expect(rig.store.getRecord('session-r')?.lease).toMatchObject({ claimStatus: 'released' }) + expect(rig.store.getRecord('session-c')?.lease.claimStatus).toBe('conflicted') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-equality.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-equality.ts new file mode 100644 index 000000000000..6efb7c02fcdb --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-equality.ts @@ -0,0 +1,52 @@ +// When two status projections of one session are the same row to a reader, so a re-projection +// that changes nothing a list shows is not re-sent. + +import { agentProviderSessionsEqual } from '../../../shared/agent-session-resume' +import { isAgentStatusHeldOpenByChildWork } from '../../../shared/agent-lead-status-fold' +import { + agentSessionBackgroundTasksEqual, + type AgentSessionStatusSummary +} from '../../../shared/agent-session-wire' +import { structuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status' + +export function structuredAgentSessionSummariesEqual( + a: AgentSessionStatusSummary, + b: AgentSessionStatusSummary +): boolean { + return ( + a.workspaceId === b.workspaceId && + a.agent === b.agent && + a.status === b.status && + a.hostExecutionOwned === b.hostExecutionOwned && + a.hostExecutionPhase === b.hostExecutionPhase && + a.rewindBlockedReason === b.rewindBlockedReason && + // A moved state clock changes ranking; row activity alone, including a subagent's, does not. + // An idle state the journal cannot date still republishes, since readers date it by `updatedAt`, + // and so does one live child work holds open: readers take each publish as its evidence. + a.statusStartedAt === b.statusStartedAt && + (a.status !== 'idle' || + a.updatedAt === b.updatedAt || + (a.statusStartedAt !== undefined && !isIdleHeldOpenByChildWork(b))) && + a.latestPrompt === b.latestPrompt && + a.model === b.model && + a.toolName === b.toolName && + a.toolInput === b.toolInput && + a.lastAssistantMessage === b.lastAssistantMessage && + a.turnOutcome === b.turnOutcome && + agentSessionBackgroundTasksEqual(a.backgroundTasks, b.backgroundTasks) && + agentProviderSessionsEqual(undefined, a.providerSession, b.providerSession) + ) +} + +function isIdleHeldOpenByChildWork(summary: AgentSessionStatusSummary): boolean { + return ( + summary.status === 'idle' && + isAgentStatusHeldOpenByChildWork( + structuredAgentSessionAgentStatus({ + status: summary.status, + backgroundTasks: summary.backgroundTasks, + turnOutcome: summary.turnOutcome + }) + ) + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts index e4fa6ebed1bf..c130aaf9d597 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts @@ -2,17 +2,11 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store export function indexedStatusFeedSession(session: { journal: AgentSessionJournal - hasProviderChild?: boolean - providerChildPhase?: 'starting' | 'ready' - fence?: number + child?: { phase: 'starting' | 'ready' } | null }) { return { journal: session.journal, - fence: session.fence ?? 1, - ...(session.hasProviderChild !== undefined - ? { hasProviderChild: session.hasProviderChild } - : {}), - ...(session.providerChildPhase ? { providerChildPhase: session.providerChildPhase } : {}), + ...(session.child !== undefined ? { child: session.child } : {}), params: { location: { executionHostId: 'local' as const, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index 65c7f54fb305..e5da2eece97c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' import type { AgentSessionBackgroundTask, AgentSessionStatusEvent, @@ -33,6 +34,8 @@ const USER_IDENTITY = { ordinal: 1 } as const +type Indexed = Parameters[0] + let root: string const journals = createTrackedJournalOpener() @@ -82,7 +85,8 @@ function feedFor( } } } as unknown as ReadonlyMap>, - getRecord: () => record as AgentSessionRecord | null, + // A partial record still has a lease: the feed reads the conversation's fence off it. + getRecord: () => (record ? { ...agentSessionRecordFixture(), ...record } : null), now: () => (now += 1) }) const events: AgentSessionStatusEvent[] = [] @@ -93,17 +97,17 @@ function feedFor( describe('StructuredAgentSessionStatusFeed', () => { it('projects whether the owned child has proven its start, and nothing once it is not owned', async () => { const journal = await openJournal() - const session = { journal, hasProviderChild: true, providerChildPhase: 'starting' as const } + const session = { journal, child: { phase: 'starting' as const } } const sessions = new Map[0]>([[SESSION, session]]) const { feed, events, dispose } = feedFor(sessions) expect(events.at(-1)).toMatchObject({ type: 'snapshot', sessions: [{ hostExecutionOwned: true, hostExecutionPhase: 'starting' }] }) - sessions.set(SESSION, { ...session, providerChildPhase: 'ready' }) + sessions.set(SESSION, { ...session, child: { phase: 'ready' } }) feed.publish(SESSION, journal) expect(events.at(-1)).toMatchObject({ session: { hostExecutionPhase: 'ready' } }) - sessions.set(SESSION, { ...session, hasProviderChild: false }) + sessions.set(SESSION, { ...session, child: null }) feed.publish(SESSION, journal) expect(events.at(-1)).not.toMatchObject({ session: { hostExecutionPhase: expect.any(String) } }) dispose() @@ -111,7 +115,7 @@ describe('StructuredAgentSessionStatusFeed', () => { it('publishes provider ownership transitions without changing journal time', async () => { const journal = await openJournal() - const sessions = new Map([[SESSION, { journal, hasProviderChild: true }]]) + const sessions = new Map([[SESSION, { journal, child: { phase: 'ready' } }]]) const { feed, events, dispose } = feedFor(sessions) events.length = 0 await journal.appendItem( @@ -130,7 +134,7 @@ describe('StructuredAgentSessionStatusFeed', () => { throw new Error('status publication missing') } const journalTime = firstStatus.session.updatedAt - sessions.get(SESSION)!.hasProviderChild = false + sessions.get(SESSION)!.child = null feed.publish(SESSION, journal) expect(events.at(-1)).toEqual({ type: 'status', @@ -167,8 +171,10 @@ describe('StructuredAgentSessionStatusFeed', () => { it('stops projecting an old-host unknown submission after the owner fence advances', async () => { const journal = await openJournal() - const session = { journal, fence: 1 } - const { feed, events } = feedFor(new Map([[SESSION, session]])) + // The conversation's fence is the record's: a child's end moves it. + const lease = agentSessionRecordFixture().lease + const record = agentSessionRecordFixture({ ...lease, runtimeFence: 1 }) + const { feed, events } = feedFor(new Map([[SESSION, { journal }]]), record) await journal.appendSubmission({ clientMessageId: 'old-host', payloadFingerprint: 'fp', @@ -183,7 +189,7 @@ describe('StructuredAgentSessionStatusFeed', () => { }) feed.publish(SESSION) expect(events.at(-1)).toMatchObject({ session: { status: 'working' } }) - session.fence = 2 + record.lease.runtimeFence = 2 feed.publish(SESSION) expect(events.at(-1)).toMatchObject({ session: { status: 'idle' } }) }) @@ -793,7 +799,7 @@ describe('the status sink sees the roster the broadcast cache deliberately lacks it('receives every change once, ownership revocation, and the forget edge', async () => { const journal = await openJournal() - const sessions = new Map([[SESSION, { journal, hasProviderChild: true }]]) + const sessions = new Map([[SESSION, { journal, child: { phase: 'ready' } }]]) const { sink, published, forgotten } = sinkFor() const { feed } = feedFor(sessions, null, undefined, undefined, sink) await journal.appendItem( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index eef1c0e06f98..d6fc7f4eb9cf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -7,24 +7,30 @@ // // The last projection is kept after the session's provider child is evicted: an idle session is // still idle without a process, and a renderer that reloads must not lose every settled row until -// each chat is reopened. Restart is the one boundary that forgets, and restoring readable sessions -// republishes them. +// each chat is reopened. A settled projection is also saved with the journal position it was +// computed at, so a restart republishes a listed chat from that saved copy when its journal still +// stands there, and from the journal otherwise. -import { agentProviderSessionsEqual } from '../../../shared/agent-session-resume' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { normalizeOptionalField } from '../../../shared/agent-status-field-normalization' -import { isAgentStatusHeldOpenByChildWork } from '../../../shared/agent-lead-status-fold' import { AGENT_MODEL_MAX_LENGTH } from '../../../shared/agent-status-types' -import { - agentSessionBackgroundTasksEqual, - type AgentSessionBackgroundTaskState, - type AgentSessionStatusEvent, - type AgentSessionStatusSummary +import type { + AgentSessionBackgroundTaskState, + AgentSessionStatusEvent, + AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import type { AgentChildWorkEvidence } from '../../../shared/agent-status-child-work-evidence' -import { projectStructuredAgentSessionStatusSummary } from '../../../shared/structured-agent-session-projection' -import { structuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status' +import { + projectStructuredAgentSessionStatusSummary, + type StructuredAgentSessionStatusProjection +} from '../../../shared/structured-agent-session-projection' +import { + isSavableStructuredAgentSessionProjection, + STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, + type StructuredAgentSessionSavedStatus +} from '../../../shared/structured-agent-session-saved-status' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { structuredAgentSessionSummariesEqual as summariesEqual } from './structured-agent-session-status-equality' import type { StructuredAgentSessionProviderChildPhase } from './structured-agent-session-adapter' import { structuredAgentSessionProviderSessionMetadata } from './structured-agent-session-history-result' import { @@ -42,9 +48,7 @@ export type StructuredAgentSessionStatusSubscriber = { type StatusFeedSession = { journal: AgentSessionJournal params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } - hasProviderChild?: boolean - providerChildPhase?: StructuredAgentSessionProviderChildPhase - fence?: number + child?: { phase: StructuredAgentSessionProviderChildPhase } | null } export type StructuredAgentSessionStatusFeedDeps = { @@ -60,45 +64,10 @@ export type StructuredAgentSessionStatusFeedDeps = { /** Live provider-owned background tasks for the summary, so session lists can * render subagent children. Optional: a provider without the hook projects none. */ readBackgroundTasks?: (sessionId: string) => AgentSessionBackgroundTaskState | null | undefined -} - -function summariesEqual(a: AgentSessionStatusSummary, b: AgentSessionStatusSummary): boolean { - return ( - a.workspaceId === b.workspaceId && - a.agent === b.agent && - a.status === b.status && - a.hostExecutionOwned === b.hostExecutionOwned && - a.hostExecutionPhase === b.hostExecutionPhase && - a.rewindBlockedReason === b.rewindBlockedReason && - // A moved state clock changes ranking; row activity alone, including a subagent's, does not. - // An idle state the journal cannot date still republishes, since readers date it by `updatedAt`, - // and so does one live child work holds open: readers take each publish as its evidence. - a.statusStartedAt === b.statusStartedAt && - (a.status !== 'idle' || - a.updatedAt === b.updatedAt || - (a.statusStartedAt !== undefined && !isIdleHeldOpenByChildWork(b))) && - a.latestPrompt === b.latestPrompt && - a.model === b.model && - a.toolName === b.toolName && - a.toolInput === b.toolInput && - a.lastAssistantMessage === b.lastAssistantMessage && - a.turnOutcome === b.turnOutcome && - agentSessionBackgroundTasksEqual(a.backgroundTasks, b.backgroundTasks) && - agentProviderSessionsEqual(undefined, a.providerSession, b.providerSession) - ) -} - -function isIdleHeldOpenByChildWork(summary: AgentSessionStatusSummary): boolean { - return ( - summary.status === 'idle' && - isAgentStatusHeldOpenByChildWork( - structuredAgentSessionAgentStatus({ - status: summary.status, - backgroundTasks: summary.backgroundTasks, - turnOutcome: summary.turnOutcome - }) - ) - ) + /** The session's agent proved a start: its row's phase became `ready`. */ + onAgentStarted?: (sessionId: string) => void + /** A settled projection and the journal position it was computed at, for the next restart. */ + saveProjection?: (sessionId: string, saved: StructuredAgentSessionSavedStatus) => void } /** Wire the host's own deps into a feed; keeps the host at one call site. @@ -116,7 +85,9 @@ export function createStructuredAgentSessionHostStatusFeed(args: { } onSessionStatusChanged?: StructuredAgentSessionStatusFeedDeps['onStatusChanged'] statusSink?: StructuredAgentSessionStatusSink + savedStatus?: { record: NonNullable } } + onAgentStarted?: (sessionId: string) => void }): StructuredAgentSessionStatusFeed { return new StructuredAgentSessionStatusFeed({ sessions: args.sessions, @@ -126,7 +97,9 @@ export function createStructuredAgentSessionHostStatusFeed(args: { readBackgroundTasks: (sessionId) => args.deps().adapter.backgroundTaskState?.(sessionId), // Resolved per call for the same reason the other deps are: the host builds this feed in a // field initializer, before its constructor parameters are assigned. - statusSink: () => args.deps().statusSink + statusSink: () => args.deps().statusSink, + saveProjection: (sessionId, saved) => args.deps().savedStatus?.record(sessionId, saved), + ...(args.onAgentStarted ? { onAgentStarted: args.onAgentStarted } : {}) }) } @@ -223,8 +196,31 @@ export class StructuredAgentSessionStatusFeed { this.published.set(sessionId, summary) this.sink(summary, session.params.location) this.broadcast({ type: 'status', session: summary }) + if (summary.hostExecutionPhase === 'ready' && previous?.hostExecutionPhase !== 'ready') { + this.deps.onAgentStarted?.(sessionId) + } + this.notify(summary, options?.replay === true) + } + + /** A restarted host's row for a chat it has not opened, from a saved copy the caller proved + * current. A replay: it is state the host already had, so no rename or restart edge fires. */ + seedRestored(sessionId: string, saved: StructuredAgentSessionSavedStatus): boolean { + const record = this.deps.getRecord(sessionId) + if (!record || this.deps.sessions.has(sessionId) || this.published.has(sessionId)) { + return false + } + const target = { params: { location: record.location, provider: record.provider } } + const summary = this.compose(sessionId, target, saved.projection, saved.lastActivityAt) + this.published.set(sessionId, summary) + this.sink(summary, record.location) + this.broadcast({ type: 'status', session: summary }) + this.notify(summary, true) + return true + } + + private notify(summary: AgentSessionStatusSummary, replay: boolean): void { try { - this.deps.onStatusChanged?.(summary, { replay: options?.replay === true }) + this.deps.onStatusChanged?.(summary, { replay }) } catch (error) { // An observer must never cost the subscribers their status event. console.warn('[structured-session-status] status observer failed', error) @@ -239,7 +235,9 @@ export class StructuredAgentSessionStatusFeed { // An unreadable journal projects as "no turn": the chat itself shows the reset. const cursor = journal.cursor() const readOnly = journal.isReadOnly - const fence = session.fence + const record = this.deps.getRecord(sessionId) + // The conversation's fence, which a child's end moves: its unanswered sends stop counting. + const fence = record?.lease.runtimeFence let projection = this.journalProjections.get(journal) if ( !projection || @@ -262,8 +260,36 @@ export class StructuredAgentSessionStatusFeed { ) } this.journalProjections.set(journal, projection) + if (!readOnly && isSavableStructuredAgentSessionProjection(projection.summary)) { + this.save(sessionId, { + v: STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, + cursor, + projection: projection.summary, + lastActivityAt: journal.lastActivityAt() + }) + } } - const record = this.deps.getRecord(sessionId) + return this.compose(sessionId, session, projection.summary, journal.lastActivityAt(), record) + } + + /** Best-effort: a failing save costs the next restart one journal open, never this publish. */ + private save(sessionId: string, saved: StructuredAgentSessionSavedStatus): void { + try { + this.deps.saveProjection?.(sessionId, saved) + } catch (error) { + console.warn('[structured-session-status] saving the settled status failed', error) + } + } + + /** The one composer for a journal projection and a saved one: record-side fields are read here, + * never saved, so a model switched while idle cannot leave two copies that disagree. */ + private compose( + sessionId: string, + session: Pick, + projection: StructuredAgentSessionStatusProjection, + lastActivityAt: number, + record = this.deps.getRecord(sessionId) + ): AgentSessionStatusSummary { const providerSession = structuredAgentSessionProviderSessionMetadata(record) // The journal has no model: the record's acknowledged options are where a mid-session // switch lands, so the row follows whichever is in force. @@ -278,22 +304,17 @@ export class StructuredAgentSessionStatusFeed { sessionId, workspaceId: session.params.location.workspaceId, agent: session.params.provider, - ...(session.hasProviderChild - ? { - hostExecutionOwned: true as const, - ...(session.providerChildPhase - ? { hostExecutionPhase: session.providerChildPhase } - : {}) - } + ...(session.child + ? { hostExecutionOwned: true as const, hostExecutionPhase: session.child.phase } : {}), - ...projection.summary, + ...projection, ...(record?.rewind?.phase === 'prepared' || record?.rewind?.phase === 'provider-succeeded' ? { rewindBlockedReason: 'outcome-unknown' as const } : {}), ...(model ? { model } : {}), ...(backgroundTasks && backgroundTasks.length > 0 ? { backgroundTasks } : {}), ...(providerSession ? { providerSession } : {}), - updatedAt: journal.lastActivityAt() || this.deps.now() + updatedAt: lastActivityAt || this.deps.now() } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts index cf9089326671..4053988233ff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts @@ -63,7 +63,9 @@ async function openSession() { const roster: { tasks: AgentSessionBackgroundTask[] } = { tasks: [] } const server = new AgentHookServer() const feed = new StructuredAgentSessionStatusFeed({ - sessions: new Map([[SESSION, indexedStatusFeedSession({ journal, hasProviderChild: true })]]), + sessions: new Map([ + [SESSION, indexedStatusFeedSession({ journal, child: { phase: 'ready' } })] + ]), getRecord: () => null, now: () => 1, readBackgroundTasks: () => ({ state: 'monitoring', tasks: roster.tasks }), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index 0b54ea7c4309..f600d6dd9bba 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -58,6 +58,10 @@ export class AgentSessionSubscribers { return this.activityBySession.size } + subscriberCountForTests(sessionId: string): number { + return this.bySession.get(sessionId)?.size ?? 0 + } + open(input: { id: string sessionId: string diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index 09dd5f82197d..dfeb93fc7099 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -1,7 +1,8 @@ -// The lifetime of a provider child, from the surfaces that hold the session. +// The lifetime of a provider child, against the real host rather than a double. // -// Two leaks meet here and each has to be tested against the real host, not a double: a chat that -// closes without stopping its app-server, and a launch that starts one for every record on disk. +// Two leaks meet here: a chat that closes without stopping its app-server, and a launch that +// starts one for every record on disk. A view never starts or keeps a child; work starts one, the +// idle sweep stops it, and an exit is settled and left for the next send. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -16,7 +17,6 @@ import type { AgentSessionMutationEnvelope, AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' -import { AGENT_SESSION_UNATTACHED_REFUSAL_CODE } from '../../../shared/structured-agent-session-read-refusal' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { AgentSessionAcquisitionRootExitObservedError, @@ -37,9 +37,9 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } -const SURFACE = 'desktop-chat:1' -/** Short enough to keep the suite fast, long enough that an eviction is a decision and not a race. */ -const GRACE_MS = 5 +/** Short enough to keep the suite fast; the host clock below decides what is idle. */ +const SWEEP_MS = 5 +const IDLE_MS = 1_000 let root: string let store: AgentSessionRecordStore @@ -50,6 +50,7 @@ let dispatch: Mock let sink: StructuredAgentSessionEventSink | null let hostErrors: unknown[] let statusSink: StructuredAgentSessionStatusSink +let clock: number function adapter(): StructuredAgentSessionAdapter { return { acquire, @@ -71,8 +72,8 @@ function openHost( journalRoot: root, claimKeyId: 'key-1', mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, - releaseGraceMs: GRACE_MS, - now: () => NOW, + idleSweep: { intervalMs: SWEEP_MS, idleMs: IDLE_MS }, + now: () => clock, onEventSinkError: ({ error }) => hostErrors.push(error), statusSink, ...(probeOwner ? { probeOwner } : {}) @@ -92,6 +93,12 @@ async function attach(): Promise { expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) } +/** What a send's delivery or `agentSession.ensure` does: attach at the record's current fence. */ +async function startAgent(): Promise { + const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? null + expect(await host.attach(CALLER, hostTestAttachParams(fence))).toMatchObject({ ok: true }) +} + function envelope(method: string, fields: Record): AgentSessionMutationEnvelope { return { sessionId: SESSION, @@ -112,7 +119,7 @@ function emitTurnLifecycle(state: 'running' | 'completed', ordinal: number): voi ) } -/** Eviction is a sequence, not an event: the child stops first and the session is forgotten last. */ +/** The sweep stops the child first and closes the conversation last. */ function waitForEviction(): Promise { return vi.waitFor(() => { expect(closeSession).toHaveBeenCalledWith(SESSION) @@ -120,9 +127,9 @@ function waitForEviction(): Promise { }) } -/** Long enough for several grace windows to elapse, so "not evicted" means the clock declined. */ -function waitOutSeveralGraceWindows(): Promise { - return new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) +/** Long enough for many sweep ticks, so "not stopped" means the sweep declined. */ +function waitOutSeveralSweeps(): Promise { + return new Promise((resolve) => setTimeout(resolve, SWEEP_MS * 20)) } /** Fails the next eviction at `drain-published`, which leaves the session indexed for a retry. */ @@ -188,6 +195,7 @@ beforeEach(async () => { resetHostTestOperationIds() sink = null hostErrors = [] + clock = NOW statusSink = { publish: vi.fn(), forget: vi.fn() } let generation = 0 acquire = vi.fn(async ({ fence, spawnToken, events }) => { @@ -218,13 +226,13 @@ afterEach(async () => { }) describe('a chat that closes', () => { - it('releases the provider child it was holding', async () => { + it('stops the provider child it started', async () => { await attach() - await host.hold(SESSION, SURFACE) - host.release(SESSION, SURFACE) + await host.close(SESSION) - await waitForEviction() + expect(closeSession).toHaveBeenCalledWith(SESSION) + expect(host.hasSession(SESSION)).toBe(false) expect(hostErrors).toEqual([]) // The record and its journal stay; only the process and the claim on it go. expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -234,49 +242,24 @@ describe('a chat that closes', () => { }) }) - it('keeps the child while another surface still holds the session', async () => { - await attach() - await host.hold(SESSION, SURFACE) - await host.hold(SESSION, 'paired-phone:1') - - host.release(SESSION, SURFACE) - await waitOutSeveralGraceWindows() - - expect(closeSession).not.toHaveBeenCalled() - expect(host.hasSession(SESSION)).toBe(true) - }) - // The pane outlives the close by a few frames — a workspace delete closes the chats inside it - // while their panes are still mounted — so whatever a read raises in that window is what the user - // sees. This is the code the client narrows on to keep that window off the pane; a host that - // starts raising a different one there puts the red error back. - it('answers a read from the pane that outlived it with the code the client treats as transitional', async () => { + // while their panes are still mounted. A read in that window reopens the conversation as a cache + // and is answered, never refused; it starts nothing. + it('answers a read from the pane that outlived it without starting a child', async () => { await attach() - await host.hold(SESSION, SURFACE) await host.close(SESSION) - expect(host.hasSession(SESSION)).toBe(false) - expect(() => host.history({ sessionId: SESSION, direction: 'tail' })).toThrow( - AGENT_SESSION_UNATTACHED_REFUSAL_CODE - ) - expect(() => - host.subscribe({ id: 'sub-1', sessionId: SESSION, emit: () => undefined }) - ).toThrow(AGENT_SESSION_UNATTACHED_REFUSAL_CODE) - }) - - it('does not lose the session to a release the client sent twice', async () => { - await attach() - await host.hold(SESSION, SURFACE) - await host.hold(SESSION, 'paired-phone:1') - - // A retried release must retire ONE holder, which is what a set gets right and a count does not. - host.release(SESSION, SURFACE) - host.release(SESSION, SURFACE) - await waitOutSeveralGraceWindows() - expect(closeSession).not.toHaveBeenCalled() - expect(host.hasSession(SESSION)).toBe(true) + expect((await host.history({ sessionId: SESSION, direction: 'tail' })).ok).toBe(true) + const unsubscribe = await host.subscribe({ + id: 'sub-1', + sessionId: SESSION, + emit: () => undefined + }) + unsubscribe() + expect(acquire).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') }) it('answers a compatibility wait with what eviction recorded', async () => { @@ -294,6 +277,8 @@ describe('a chat that closes', () => { if (!result.ok) { throw new Error('send was refused') } + // Handed over first: a message still queued at close is rejected as never sent instead. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalled()) const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId) await host.close(SESSION) @@ -306,7 +291,7 @@ describe('a chat that closes', () => { }) }) - it('retries teardown after journal close loses its result', async () => { + it('keeps the stop it made when the journal close loses its result', async () => { await attach() const session = host['sessions'].get(SESSION) expect(session).toBeDefined() @@ -318,27 +303,16 @@ describe('a chat that closes', () => { }) .mockImplementation(closeJournal) - await expect(host.close(SESSION)).rejects.toMatchObject({ - step: 'forget-session', - cause: expect.objectContaining({ message: 'journal close result lost' }) - }) - expect(host.hasSession(SESSION)).toBe(true) - expect(host['sessions'].get(SESSION)?.hasProviderChild).toBe(false) + // The child is stopped and the lease released before the handle closes; the entry is dropped + // before that close, so a lost result leaves no closing handle for a reader to find. + await expect(host.close(SESSION)).rejects.toThrow('journal close result lost') + expect(host.hasSession(SESSION)).toBe(false) expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', ownerProcess: null }) - expect(statusSink.forget).toHaveBeenCalledWith({ - kind: 'structured-session', - sessionId: SESSION, - executionHostId: 'local', - wslDistro: null, - workspaceId: 'workspace-1', - workspaceKind: 'git-worktree' - }) await expect(host.close(SESSION)).resolves.toBeUndefined() - expect(host.hasSession(SESSION)).toBe(false) expect(closeSession).toHaveBeenCalledOnce() }) @@ -361,7 +335,7 @@ describe('a chat that closes', () => { await expect(host.close(SESSION)).rejects.toMatchObject({ step: 'drain-published' }) // The child is proven gone, but the wind-down it owes is not done: nothing settled, no release. - expect(session!.hasProviderChild).toBe(false) + expect(session!.child).toBeNull() expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') await expect(host.close(SESSION)).resolves.toBeUndefined() @@ -375,31 +349,31 @@ describe('a chat that closes', () => { }) describe('a session with a turn in flight', () => { - it('is not evicted while the turn runs, and is once it ends', async () => { + it('is not stopped while the turn runs, and is once it ends and idles', async () => { await attach() - await host.hold(SESSION, SURFACE) emitTurnLifecycle('running', 1) await host.flushStreamedEvents(SESSION) - host.release(SESSION, SURFACE) - await waitOutSeveralGraceWindows() + clock += IDLE_MS + await waitOutSeveralSweeps() expect(closeSession).not.toHaveBeenCalled() expect(host.hasSession(SESSION)).toBe(true) emitTurnLifecycle('completed', 2) await host.flushStreamedEvents(SESSION) + clock += IDLE_MS await waitForEviction() }) - // Codex settles an admitted send only on its echo, which may never come; eviction retires it. - it('is evicted with an admitted send outstanding once no turn runs', async () => { + // Codex settles an admitted send only on its echo, which may never come; the stop retires it. + it('is stopped with an admitted send outstanding once no turn runs', async () => { await attach() - await host.hold(SESSION, SURFACE) await sendPending('admitted, never echoed') + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) - host.release(SESSION, SURFACE) + clock += IDLE_MS await waitForEviction() }) @@ -408,17 +382,19 @@ describe('a session with a turn in flight', () => { describe('startup', () => { it('settles an idle absent owner without chat pollution and resumes the same provider identity', async () => { await attach() + // Listed: the startup pass settles what a listed chat owes. + await store.setSessionTabVisibility(SESSION, true) const beforeRestart = store.getRecord(SESSION) host['runtimeState'].stopLeaseRenewal() - host['holds'].dispose() + host['lifetime'].dispose() await host['sessions'].get(SESSION)?.journal.close() host['sessions'].clear() store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) openHost(async () => ({ outcome: 'pid-absent' })) - await host.restoreReadableSessions() + await host.restoreStartupSessions() - const restored = host.history({ sessionId: SESSION, direction: 'tail' }) + const restored = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(restored.ok && restored.page.items.some((item) => item.body.kind === 'status')).toBe( false ) @@ -428,7 +404,7 @@ describe('startup', () => { settlementRetryRequired: undefined }) - await host.hold(SESSION, SURFACE) + await startAgent() expect(store.getRecord(SESSION)?.providerHandleChain.at(-1)?.handle).toEqual( beforeRestart?.providerHandleChain.at(-1)?.handle ) @@ -439,23 +415,30 @@ describe('startup', () => { await attach() await reboot() - await host.restoreReadableSessions() + await host.restoreStartupSessions() // The record is readable — the tab comes back, history answers — and nothing is running. expect(acquire).not.toHaveBeenCalled() - expect(host.listSessionTabs()).toEqual([ + expect(host.listPersistedSessionTabs([SESSION])).toEqual([ { sessionId: SESSION, workspaceId: 'workspace-1', agent: 'codex' } ]) - expect(host.history({ sessionId: SESSION, direction: 'tail' }).ok).toBe(true) + expect((await host.history({ sessionId: SESSION, direction: 'tail' })).ok).toBe(true) expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') }) - it('gives the child back to a chat a surface actually opens', async () => { + it('gives the child back for work, never for a read', async () => { await attach() await reboot() - await host.restoreReadableSessions() + await host.restoreStartupSessions() - await host.hold(SESSION, SURFACE) + const unsubscribe = await host.subscribe({ + id: 'viewer-1', + sessionId: SESSION, + emit: () => undefined + }) + expect(acquire).not.toHaveBeenCalled() + await startAgent() + unsubscribe() expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -466,16 +449,15 @@ describe('startup', () => { }) }) -describe('a session evicted and opened again', () => { +describe('a session closed and started again', () => { it('publishes provider events to the reattached chat', async () => { await attach() - await host.hold(SESSION, SURFACE) - host.release(SESSION, SURFACE) - await waitForEviction() + await host.close(SESSION) + expect(host.hasSession(SESSION)).toBe(false) - await host.hold(SESSION, 'desktop-chat:2') + await startAgent() const events: AgentSessionSubscribeEvent[] = [] - const unsubscribe = host.subscribe({ + const unsubscribe = await host.subscribe({ id: 'subscriber-1', sessionId: SESSION, emit: (event) => events.push(event) @@ -492,6 +474,12 @@ describe('a session evicted and opened again', () => { }) }) +async function submissionState(clientMessageId: string): Promise { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + )?.dispatchState +} + describe('an unexpected provider exit', () => { it('publishes terminal settlement to a waiting older client', async () => { await attach() @@ -508,6 +496,8 @@ describe('an unexpected provider exit', () => { if (!result.ok) { throw new Error('send was refused') } + // Accepted first; the exit must meet a message the provider was handed. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -531,7 +521,7 @@ describe('an unexpected provider exit', () => { await failJournalSinkUntilReleased() expect(dispatch).not.toHaveBeenCalled() - const history = host.history({ sessionId: SESSION, direction: 'tail' }) + const history = await host.history({ sessionId: SESSION, direction: 'tail' }) expect( history.ok && history.page.items.some( @@ -553,17 +543,18 @@ describe('an unexpected provider exit', () => { replaceFailedSink() }) - it('releases the exact generation, reacquires outside the queue, and dispatches a new message', async () => { + it('releases the exact generation, starts nothing, and the next message starts a child', async () => { await attach() - await host.hold(SESSION, SURFACE) dispatch.mockRejectedValueOnce(new Error('provider delivery became unknown')) const unknownBody = hostTestMessage('message with unknown delivery') + const unknownEnvelope = envelope('agentSession.send', { body: unknownBody }) await expect( - host.send(CALLER, { - envelope: envelope('agentSession.send', { body: unknownBody }), - body: unknownBody - }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) + host.send(CALLER, { envelope: unknownEnvelope, body: unknownBody }) + ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // Accepted, then handed over by the delivery loop, where the thrown dispatch becomes doubt. + await vi.waitFor(async () => + expect(await submissionState(unknownEnvelope.clientOperationId)).toBe('unknown') + ) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 await host.handleAdapterEvent({ @@ -575,32 +566,42 @@ describe('an unexpected provider exit', () => { acquisitionGeneration: 'generation-1' }) - const recoveredHistory = host.history({ sessionId: SESSION, direction: 'tail' }) + const recoveredHistory = await host.history({ sessionId: SESSION, direction: 'tail' }) expect( recoveredHistory.ok && hasUnansweredStructuredAgentSessionDispatch(recoveredHistory.page.submissions) ).toBe(false) - expect(acquire).toHaveBeenCalledTimes(2) + // No respawn: the exit is settled and shown, and the conversation waits for work. + expect(acquire).toHaveBeenCalledOnce() expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ - claimStatus: 'live', - runtimeFence: exitedFence + 2, - ownerProcess: { pid: 4242 } + claimStatus: 'released', + runtimeFence: exitedFence + 1 }) dispatch.mockResolvedValueOnce({ state: 'accepted', providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('a distinct next message') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const nextEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(async () => + expect(await submissionState(nextEnvelope.clientOperationId)).toBe('accepted') + ) + expect(acquire).toHaveBeenCalledTimes(2) expect(dispatch).toHaveBeenCalledTimes(2) }) - it('does not reacquire for a subscription-only hold or a stale child generation', async () => { + it('does not reacquire for a live reader or a stale child generation', async () => { await attach() - await host.hold(SESSION, 'subscriber-1', { resume: false }) + const unsubscribe = await host.subscribe({ + id: 'subscriber-1', + sessionId: SESSION, + emit: () => undefined + }) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 await host.handleAdapterEvent({ @@ -628,11 +629,11 @@ describe('an unexpected provider exit', () => { runtimeFence: exitedFence + 1, deathEvidence: { kind: 'exit-observed' } }) + unsubscribe() }) it('keeps a requested close out of recovery', async () => { await attach() - await host.hold(SESSION, SURFACE) const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 await host.handleAdapterEvent({ @@ -650,7 +651,6 @@ describe('an unexpected provider exit', () => { it('recovers after a failed lifecycle barrier and dispatches a distinct next message', async () => { await attach() - await host.hold(SESSION, SURFACE) dispatch.mockRejectedValueOnce(new Error('provider delivery became unknown')) const unknownBody = hostTestMessage('message with unknown delivery') const unknownParams = { @@ -659,8 +659,11 @@ describe('an unexpected provider exit', () => { } await expect(host.send(CALLER, unknownParams)).resolves.toMatchObject({ ok: true, - value: { submission: { dispatchState: 'unknown' } } + value: { submission: { dispatchState: 'pending' } } }) + await vi.waitFor(async () => + expect(await submissionState(unknownParams.envelope.clientOperationId)).toBe('unknown') + ) const runtimeState = ( host as unknown as { runtimeState: { lifecycleBarrier: () => Promise<{ ok: false; error: Error }> } @@ -682,14 +685,13 @@ describe('an unexpected provider exit', () => { }) expect(store.getRecord(SESSION)?.lease).toMatchObject({ - claimStatus: 'live', - runtimeFence: exitedFence + 2, - ownerProcess: { pid: 4242 } + claimStatus: 'released', + runtimeFence: exitedFence + 1 }) - expect(acquire).toHaveBeenCalledTimes(2) + expect(acquire).toHaveBeenCalledOnce() expect(dispatch).toHaveBeenCalledOnce() expect(hostErrors).toContainEqual(expect.objectContaining({ message: 'journal failed' })) - const history = host.history({ sessionId: SESSION, direction: 'tail' }) + const history = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(history.ok && history.page.submissions[0]?.dispatchState).toBe('unknown') // A send whose delivery outcome is unknown IS work in progress, so the reassuring outcome is // written — carrying the cause, and never the old bare `Provider exited: ` row. @@ -704,15 +706,19 @@ describe('an unexpected provider exit', () => { providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('a distinct next message after failed-barrier recovery') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const nextEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(async () => + expect(await submissionState(nextEnvelope.clientOperationId)).toBe('accepted') + ) expect(dispatch).toHaveBeenCalledTimes(2) }) it('latches a failed exit settlement and blocks attach until the terminal batch is written', async () => { await attach() - await host.hold(SESSION, SURFACE) emitTurnLifecycle('running', 1) await host.flushStreamedEvents(SESSION) const runtimeState = ( @@ -785,7 +791,7 @@ describe('a quit over an eviction that never got its retry', () => { failNextDrain() await expect(host.close(SESSION)).rejects.toMatchObject({ step: 'drain-published' }) - expect(host['sessions'].get(SESSION)?.hasProviderChild).toBe(false) + expect(host['sessions'].get(SESSION)?.child).toBeNull() expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') await host.flushAllStreamedEvents() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index 794b4fe8df90..5eac140868f8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -25,11 +25,7 @@ import type { } from './structured-agent-session-adapter' import { providerStartupFailureRejection } from './structured-agent-session-dead-generation-settlement' import { validatePendingPrompt } from './structured-agent-session-prompt-state' -import { withTimeout } from '../../../shared/promise-timeout-fallback' -import { - AgentSessionPreDispatchError, - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS -} from './structured-agent-session-operation-settlement' +import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' export { performSetOption } from './structured-agent-session-turns-options' export { performPrompt } from './structured-agent-session-turns-prompt' @@ -47,8 +43,6 @@ export type AgentSessionTurnContext = { publish: () => void /** Drains provider lifecycle already accepted by the execution host. */ flushStreamedEvents: () => Promise - /** Re-derives authorization after submission persistence, immediately before provider dispatch. */ - beforeDispatch?: () => void /** What the host holds about the child this dispatch is for, read at the moment it is needed. */ providerChildPhase?: () => StructuredAgentSessionProviderChildPhase | undefined now: () => number @@ -67,10 +61,10 @@ function invalid(message: string): { ok: false; refusal: AgentSessionWireRefusal * accepted nothing (input is written only after it initializes), so a dispatch it could not * take is provably unwritten and is rejected with the cause the adapter gave. */ async function dispatchSafely( - ctx: AgentSessionTurnContext, + ctx: AgentSessionHandoverContext, clientMessageId: string, body: AgentJournalMessageItem, - requestedAt: number | undefined + requestedAt: number ): Promise { try { return await ctx.adapter.dispatch({ @@ -78,13 +72,9 @@ async function dispatchSafely( clientMessageId, body, fence: ctx.fence, - ...(ctx.beforeDispatch ? { beforeDispatch: async () => ctx.beforeDispatch?.() } : {}), - ...(requestedAt === undefined ? {} : { requestedAt }) + requestedAt }) } catch (error) { - if (error instanceof AgentSessionPreDispatchError) { - throw error - } if (ctx.providerChildPhase?.() === 'starting') { return { state: 'rejected', reason: providerStartupFailureRejection(error) } } @@ -111,6 +101,8 @@ async function appendStatus( * the whole content of the word — and one message reached the model five times * when this was a judgement call instead of an invariant. A distinct send after * a terminal rejection uses a fresh id, which is a first delivery. + * + * Accepting only records the message; the session's delivery loop hands it over. */ export async function performSend( ctx: AgentSessionTurnContext, @@ -133,86 +125,77 @@ export async function performSend( } } try { - await ctx.journal.appendSubmission({ ...input, fence: ctx.fence }) + await ctx.journal.appendSubmission({ ...input, fence: ctx.fence, handoverRecorded: true }) } catch { return invalid('The message could not be recorded and was not sent.') } + return { + ok: true, + value: { + clientMessageId: input.clientMessageId, + submission: requireSubmission(ctx, input.clientMessageId) + } + } +} - // The row just written is the send's instant on the host clock; the turn this +export type AgentSessionHandoverContext = Pick< + AgentSessionTurnContext, + 'sessionId' | 'journal' | 'fence' | 'adapter' | 'providerChildPhase' +> + +/** + * Hands one queued submission to the provider. The `dispatch{pending}` row goes first: a crash + * after it leaves a message in doubt, never one that reads as queued and so provably unwritten. + */ +export async function handOverSubmission( + ctx: AgentSessionHandoverContext, + submission: AgentJournalSubmission +): Promise { + const { clientMessageId } = submission + const body = ctx.journal.itemBody(agentJournalSubmissionKey(clientMessageId)) + if (body?.kind !== 'message') { + await ctx.journal.resolveDispatch({ + clientMessageId, + state: 'rejected', + reason: 'The message could not be read back and was not sent.', + fence: ctx.fence + }) + return + } + await ctx.journal.resolveDispatch({ clientMessageId, state: 'pending', fence: ctx.fence }) + // The row written at acceptance is the send's instant on the host clock; the turn this // dispatch opens records it so the live counter never re-anchors at turn-open. - const requestedAt = ctx.journal - .submissions() - .find((entry) => entry.clientMessageId === input.clientMessageId)?.submittedAt - const outcome = await dispatchSafely(ctx, input.clientMessageId, input.body, requestedAt).catch( - async (error: unknown) => { - if (error instanceof AgentSessionPreDispatchError) { - const recorded = await withTimeout( - ctx.journal - .resolveDispatch({ - clientMessageId: input.clientMessageId, - state: 'rejected', - reason: error.message, - fence: ctx.fence - }) - .then(() => true), - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS, - false - ) - if (!recorded) { - console.warn('[structured-agent-session] pre-dispatch refusal persistence failed') - } - } - throw error - } - ) + const outcome = await dispatchSafely(ctx, clientMessageId, body, submission.submittedAt) // An admission needs no dispatch row: the submission is already pending. if (outcome.state === 'admitted') { - return { - ok: true, - value: { - clientMessageId: input.clientMessageId, - submission: requireSubmission(ctx, input.clientMessageId) - } - } + return } try { await ctx.journal.resolveDispatch( outcome.state === 'accepted' ? { - clientMessageId: input.clientMessageId, + clientMessageId, state: 'accepted', providerIdentity: outcome.providerIdentity, fence: ctx.fence } - : { - clientMessageId: input.clientMessageId, - state: outcome.state, - reason: outcome.reason, - fence: ctx.fence - } + : { clientMessageId, state: outcome.state, reason: outcome.reason, fence: ctx.fence } ) } catch (error) { // A failed resolution must not strand a pending row; an unknown result is // explicitly replayable. try { await ctx.journal.resolveDispatch({ - clientMessageId: input.clientMessageId, + clientMessageId, state: 'unknown', reason: DISPATCH_DOUBT_PERSISTENCE_FAILED, fence: ctx.fence }) } catch { - // Nothing further to record; the pending row is settled on the next attach. + // Nothing further to record; the pending row is settled on the next open. } throw error } - return { - ok: true, - value: { - clientMessageId: input.clientMessageId, - submission: requireSubmission(ctx, input.clientMessageId) - } - } } function requireSubmission( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts index 79f21a349b42..952fa70657d9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts @@ -32,10 +32,12 @@ function contextWith(submissions: AgentJournalSubmission[]) { return { epoch: 'e', sequence: 1 } }) } - // The mutation reads only `journal.submissions`, `journal.resolveDispatch` and `fence`. - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all three are supplied here; the rest of the host session is unreachable from this mutation. - const session = { journal, fence: FENCE } as unknown as ReleaseSession - const context: ReleaseContext = { sessions: new Map([['s-1', session]]) } + // The mutation reads only `journal.submissions`, `journal.resolveDispatch` and the record fence. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all three are supplied here; the rest of the session and the record is unreachable from this mutation. + const context = { + sessions: new Map([['s-1', { journal }]]), + deps: { store: { getRecord: () => ({ lease: { runtimeFence: FENCE } }) } } + } as unknown as ReleaseContext return { context, resolved, journal } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 4ba1a93169b0..7abaabada6cb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -10,47 +10,14 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-sessi import { unexpectedProviderExitOutcome } from './structured-agent-session-dead-generation-settlement' import { retryLoadedStructuredAgentSessionSettlement } from './structured-agent-session-settlement-retry' import { - isStructuredAgentSessionRecoveryTicketCurrent, settleUnexpectedStructuredAgentSessionExit, type StructuredAgentSessionUnexpectedExitContext, - type StructuredAgentSessionUnexpectedExitSession, - type StructuredAgentSessionRecoveryTicket + type StructuredAgentSessionUnexpectedExitSession } from './structured-agent-session-unexpected-exit' const SESSION = 'session-1' const GENERATION = 'generation-1' -const ticket: StructuredAgentSessionRecoveryTicket = { - sessionId: SESSION, - releasedFence: 8, - deadAcquisitionGeneration: GENERATION, - stableSettlementId: 'settlement-1' -} - -function recoveryContext(input: { - generation?: string - handoffStage?: AgentSessionRecord['lease']['handoffStage'] - resumeCapable?: boolean -}) { - const session = { - hasProviderChild: false, - fence: 8, - acquisitionGeneration: input.generation ?? GENERATION - } as StructuredAgentSessionHostSession - const record = { - lease: { - runtimeFence: 8, - claimStatus: 'released', - handoffStage: input.handoffStage ?? null - } - } as AgentSessionRecord - return { - sessions: new Map([[SESSION, session]]), - store: { getRecord: () => record }, - hasResumeCapableHolder: () => input.resumeCapable ?? true - } as never -} - function lifecycleItem( turnId: string, sequence: number, @@ -102,7 +69,7 @@ function mutableStore() { } } -describe('provider-exit recovery tickets', () => { +describe('provider-exit settlement', () => { it.each([undefined, 2_000])('keeps exit receipt %s on retry', async (observedAt) => { let now = observedAt === undefined ? 2_000 : 30_000 let record = { @@ -127,11 +94,11 @@ describe('provider-exit recovery tickets', () => { .fn() .mockRejectedValueOnce(new Error('journal unavailable')) .mockResolvedValue({ epoch: 'epoch-1', sequence: 2 }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit reads only the child record and these journal methods; the rest of the session is unreachable from it. const session = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })] }), @@ -149,7 +116,6 @@ describe('provider-exit recovery tickets', () => { return { ok: false, error: new Error('sink unavailable') } }, publishFence: vi.fn(), - hasResumeCapableHolder: () => true, serialize: async (_sessionId, task) => task(), now: () => now } as never, @@ -173,7 +139,7 @@ describe('provider-exit recovery tickets', () => { retryLoadedStructuredAgentSessionSettlement({ deps: { store } as never, sessionId: SESSION, - session: { journal: session.journal, fence: 8, acquisitionGeneration: null }, + journal: session.journal, now: () => now }) ).resolves.toBe(true) @@ -197,11 +163,11 @@ describe('provider-exit recovery tickets', () => { lifecycleItem('turn-1', 1, { state: 'completed', startedAt: 10, completedAt: 20 }), lifecycleItem('turn-2', 2, { state: 'running', startedAt: 30 }) ] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit reads only the child record and these journal methods; the rest of the session is unreachable from it. const session = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items }), appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []) @@ -222,13 +188,12 @@ describe('provider-exit recovery tickets', () => { transitionHandoff: async () => ({ lease: { runtimeFence: 8 } }) } - const result = await settleUnexpectedStructuredAgentSessionExit( + await settleUnexpectedStructuredAgentSessionExit( { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), publishFence: vi.fn(), - hasResumeCapableHolder: () => true, serialize: async (_sessionId, task) => task(), now: () => 1_234 } as never, @@ -243,12 +208,11 @@ describe('provider-exit recovery tickets', () => { } ) - expect(result).toMatchObject({ releasedFence: 8 }) expect(session.journal.markPendingSubmissionsUnknown).toHaveBeenCalledWith( 7, 'provider_exited_before_acknowledgement' ) - expect(session.hasProviderChild).toBe(false) + expect(session.child).toBeNull() // The running row is revised to interrupted at exit receipt, never tombstoned. expect(appendLifecycleBatch).toHaveBeenCalledExactlyOnceWith({ settlementId: `dead-generation:provider-exit:${SESSION}:7:${GENERATION}`, @@ -305,10 +269,9 @@ describe('provider-exit recovery tickets', () => { sequence: 3 })) const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items }), appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []), @@ -331,7 +294,6 @@ describe('provider-exit recovery tickets', () => { return { ok: true } }, publishFence: vi.fn(), - hasResumeCapableHolder: () => true, serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1_234 } @@ -362,10 +324,9 @@ describe('provider-exit recovery tickets', () => { it('settles a submission the dead child never acknowledged', async () => { const markPendingSubmissionsUnknown = vi.fn(async () => ['client-1']) const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items: [] }), appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), markPendingSubmissionsUnknown, @@ -380,7 +341,6 @@ describe('provider-exit recovery tickets', () => { sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), publishFence: vi.fn(), - hasResumeCapableHolder: () => true, serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1 } @@ -410,10 +370,9 @@ describe('provider-exit recovery tickets', () => { it('does not release or reacquire while terminal settlement retry is still failing', async () => { const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), markPendingSubmissionsUnknown: vi.fn(async () => []), rejectPendingSubmissions: vi.fn(async () => []), snapshot: () => ({ @@ -440,45 +399,14 @@ describe('provider-exit recovery tickets', () => { sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: false, error: new Error('sink failed') }), publishFence, - hasResumeCapableHolder: () => true, serialize: async (_sessionId, task) => task(), now: () => 1, onBarrierError: release } - const result = await settleUnexpectedStructuredAgentSessionExit(context, event) + await settleUnexpectedStructuredAgentSessionExit(context, event) - expect(result).toBeNull() - expect(session.hasProviderChild).toBe(false) - expect(session.fence).toBe(8) + expect(session.child).toBeNull() expect(publishFence).toHaveBeenCalledTimes(1) expect(release).toHaveBeenCalledTimes(2) }) - - it('admits the exact released generation for a resume-capable holder', () => { - expect(isStructuredAgentSessionRecoveryTicketCurrent(recoveryContext({}), ticket)).toBe(true) - }) - - it('is cancelled by a latched stage before reattachment', () => { - expect( - isStructuredAgentSessionRecoveryTicketCurrent( - recoveryContext({ handoffStage: 'recovering' }), - ticket - ) - ).toBe(false) - }) - - it('is cancelled when its holder or dead acquisition generation is no longer current', () => { - expect( - isStructuredAgentSessionRecoveryTicketCurrent( - recoveryContext({ resumeCapable: false }), - ticket - ) - ).toBe(false) - expect( - isStructuredAgentSessionRecoveryTicketCurrent( - recoveryContext({ generation: 'generation-new' }), - ticket - ) - ).toBe(false) - }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts index 5363b8588e74..fdc6a41d7bdd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts @@ -1,8 +1,7 @@ -import type { - StructuredAgentSessionEndedEvent, - StructuredAgentSessionProviderChildPhase -} from './structured-agent-session-adapter' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { endProviderChild } from './structured-agent-session-provider-child' import { releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit, type StructuredAgentSessionLeaseStore @@ -21,20 +20,10 @@ type UnexpectedExitLifecycleEvent = StructuredAgentSessionEndedEvent & { cause: 'unexpected-exit' } -export type StructuredAgentSessionRecoveryTicket = { - sessionId: string - releasedFence: number - deadAcquisitionGeneration: string - stableSettlementId: string -} - -export type StructuredAgentSessionUnexpectedExitSession = { - journal: DeadGenerationJournal - hasProviderChild: boolean - fence: number - acquisitionGeneration: string | null - providerChildPhase?: StructuredAgentSessionProviderChildPhase -} +export type StructuredAgentSessionUnexpectedExitSession = Pick< + StructuredAgentSessionHostSession, + 'child' | 'lastEndedChild' +> & { journal: DeadGenerationJournal & Pick } export type StructuredAgentSessionUnexpectedExitContext< TSession extends StructuredAgentSessionUnexpectedExitSession = StructuredAgentSessionHostSession @@ -44,7 +33,6 @@ export type StructuredAgentSessionUnexpectedExitContext< flushLifecycle: (sessionId: string) => Promise publishFence: (sessionId: string, session: TSession) => void publishStatus?: (sessionId: string) => void - hasResumeCapableHolder: (sessionId: string) => boolean serialize: (sessionId: string, task: () => Promise) => Promise now: () => number onBarrierError?: (sessionId: string, error: unknown) => void @@ -55,40 +43,50 @@ export async function settleUnexpectedStructuredAgentSessionExit< >( context: StructuredAgentSessionUnexpectedExitContext, event: StructuredAgentSessionEndedEvent -): Promise { +): Promise { if (event.cause !== 'unexpected-exit') { - return null + return } const unexpectedEvent = event as UnexpectedExitLifecycleEvent // Receipt of the exit is the one end time the host may record for a running turn. const observedAt = event.observedAt ?? context.now() return context.serialize(unexpectedEvent.sessionId, async () => { const session = context.sessions.get(unexpectedEvent.sessionId) + const child = session?.child if ( - !session?.hasProviderChild || - session.fence !== unexpectedEvent.fence || - session.acquisitionGeneration !== unexpectedEvent.acquisitionGeneration + !session || + !child || + child.fence !== unexpectedEvent.fence || + child.generation !== unexpectedEvent.acquisitionGeneration ) { - return null + return + } + // The host's own phase decides, so a provider that omits the flag still gets a start that + // failed told as one: the row says so. + const exitedDuringStartup = + unexpectedEvent.startupUnproven === true || child.phase === 'starting' + const endChild = (): void => { + endProviderChild(session, { + generation: child.generation, + fence: child.fence, + cause: 'exit', + reason: unexpectedEvent.reason, + duringStartup: exitedDuringStartup, + // The adapter publishes an exit only once it saw the root go, first-hand or proven. + rootGone: true + }) + context.publishStatus?.(unexpectedEvent.sessionId) } const record = context.store.getRecord(unexpectedEvent.sessionId) if (!record || record.lease.handoffStage !== null) { // An acquisition or recovery already owns this lease's transition. - session.hasProviderChild = false - context.publishStatus?.(unexpectedEvent.sessionId) - return null + endChild() + return } - // The host's own phase decides, so a provider that omits the flag still gets a start that - // failed told as one: the row says so, and nothing resumes into the same failure. - const exitedDuringStartup = - unexpectedEvent.startupUnproven === true || session.providerChildPhase === 'starting' let settlementFailed = false const stableSettlementId = providerExitSettlementId(unexpectedEvent) const unfinishedWork = captureUnfinishedStructuredAgentSessionWork(session.journal) - let released: Awaited< - ReturnType - > | null = null try { try { const barrier = await context.flushLifecycle(unexpectedEvent.sessionId) @@ -101,7 +99,8 @@ export async function settleUnexpectedStructuredAgentSessionExit< settlementFailed = !(await retryUnexpectedExitSettlement({ context, event: unexpectedEvent, - session, + journal: session.journal, + fence: child.fence, stableSettlementId, verdict: { state: 'interrupted', completedAt: observedAt }, exitedDuringStartup, @@ -117,13 +116,16 @@ export async function settleUnexpectedStructuredAgentSessionExit< } finally { // Provider exit was positively observed, so release the owner even when // terminal settlement could not be durably accepted. + let released: Awaited< + ReturnType + > | null = null try { released = await releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit({ store: context.store, sessionId: unexpectedEvent.sessionId, expectedFence: unexpectedEvent.fence, expectedAcquisitionGeneration: unexpectedEvent.acquisitionGeneration, - acquisitionGeneration: session.acquisitionGeneration, + acquisitionGeneration: child.generation, now: context.now(), exitObservedAt: observedAt, ...(settlementFailed @@ -139,76 +141,37 @@ export async function settleUnexpectedStructuredAgentSessionExit< } catch (error) { context.onBarrierError?.(unexpectedEvent.sessionId, error) } finally { - session.hasProviderChild = false - context.publishStatus?.(unexpectedEvent.sessionId) + endChild() if (released) { - session.fence = released.lease.runtimeFence context.publishFence(unexpectedEvent.sessionId, session) } } } - if (settlementFailed || !released) { - return null - } - // Resuming a start that failed would respawn into the same failure; the next send retries. - if (exitedDuringStartup || !context.hasResumeCapableHolder(unexpectedEvent.sessionId)) { - return null - } - return { - sessionId: unexpectedEvent.sessionId, - releasedFence: released.lease.runtimeFence, - deadAcquisitionGeneration: unexpectedEvent.acquisitionGeneration, - stableSettlementId - } }) } -export function isStructuredAgentSessionRecoveryTicketCurrent( - context: { - store: Pick - sessions: Map< - string, - Pick< - StructuredAgentSessionUnexpectedExitSession, - 'hasProviderChild' | 'fence' | 'acquisitionGeneration' - > - > - hasResumeCapableHolder: (sessionId: string) => boolean - }, - ticket: StructuredAgentSessionRecoveryTicket -): boolean { - const session = context.sessions.get(ticket.sessionId) - const record = context.store.getRecord(ticket.sessionId) - return ( - session?.hasProviderChild === false && - session.fence === ticket.releasedFence && - session.acquisitionGeneration === ticket.deadAcquisitionGeneration && - record?.lease.runtimeFence === ticket.releasedFence && - record.lease.claimStatus === 'released' && - record.lease.handoffStage === null && - context.hasResumeCapableHolder(ticket.sessionId) - ) -} - async function retryUnexpectedExitSettlement(input: { context: Pick event: UnexpectedExitLifecycleEvent - session: Pick + journal: DeadGenerationJournal + fence: number stableSettlementId: string verdict: StructuredAgentSessionTurnVerdict exitedDuringStartup: boolean showUnexpectedExitOutcome?: boolean }): Promise { return settleStructuredAgentSessionDeadGeneration({ - journal: input.session.journal, + journal: input.journal, sessionId: input.event.sessionId, - fence: input.session.fence, + fence: input.fence, settlementId: input.stableSettlementId, verdict: input.verdict, pendingSubmissionReason: 'provider_exited_before_acknowledgement', showUnexpectedExitOutcome: input.showUnexpectedExitOutcome, unexpectedExitReason: input.event.reason, - exitedDuringStartup: input.exitedDuringStartup, + ...(input.exitedDuringStartup + ? { exitedDuringStartup: { generation: input.event.acquisitionGeneration } } + : {}), onError: input.context.onBarrierError }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unreadable-journal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unreadable-journal.test.ts new file mode 100644 index 000000000000..bedecd17732d --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unreadable-journal.test.ts @@ -0,0 +1,42 @@ +// A chat whose journal file is not a usable database: its reads refuse with the one code a client +// treats as final, instead of a raw SQLite message the pane would re-ask forever. + +import { writeFile } from 'node:fs/promises' +import { afterEach, describe, expect, it } from 'vitest' +import { journalDatabaseFile, journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { + createRestTestRig, + foundRestTestChat, + REST_TEST_SESSION as SESSION, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +let rig: RestTestRig | null = null + +afterEach(async () => { + await rig?.dispose() + rig = null +}) + +async function corruptJournal(active: RestTestRig): Promise { + const dir = journalDirectoryFor(active.root, { workspaceId: 'workspace-1', sessionId: SESSION }) + await writeFile(journalDatabaseFile(dir), Buffer.alloc(8192, 7)) +} + +describe('an unloadable chat journal', () => { + it('refuses a history read with agent_session_journal_unreadable on every ask', async () => { + const active = await createRestTestRig() + rig = active + await foundRestTestChat(active) + await active.restart() + await corruptJournal(active) + const restarted = await active.restart() + + await expect(restarted.history({ sessionId: SESSION, direction: 'tail' })).rejects.toThrow( + /^agent_session_journal_unreadable$/ + ) + await expect(restarted.history({ sessionId: SESSION, direction: 'tail' })).rejects.toThrow( + /^agent_session_journal_unreadable$/ + ) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 21a5fe4e49d4..2a5168fce461 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -6,9 +6,9 @@ // conversation — the journal, the provider handle chain, and the recorded evidence all survive. // // "Usable" means ACQUIRABLE, not acquired. Startup no longer resumes a provider child for a record -// nobody is looking at; a surface taking a hold is what spawns one. So the migration's job is to -// leave the lease in a state a hold can claim, and these tests prove that by adjudicating it rather -// than by reading fields off it. +// nobody is looking at; work that needs the agent is what spawns one. So the migration's job is to +// leave the lease in a state an attach can claim, and these tests prove that by adjudicating it +// rather than by reading fields off it. import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -105,7 +105,10 @@ function wedgedRecord(overrides: WedgeOverrides): PersistedAgentSessionRecord { } } -async function seedStore(record: PersistedAgentSessionRecord): Promise { +async function seedStore( + record: PersistedAgentSessionRecord, + options: { listed?: boolean } = {} +): Promise { const directory = join(root, 'store') await mkdir(directory, { recursive: true }) await writeFile( @@ -116,7 +119,8 @@ async function seedStore(record: PersistedAgentSessionRecord): Promise { records: { [record.sessionId]: record }, operations: {}, retiredClaimKeys: [], - unusableRecords: {} + unusableRecords: {}, + ...(options.listed ? { visibleSessionIds: [record.sessionId] } : {}) }), 'utf-8' ) @@ -254,13 +258,15 @@ describe('already-wedged profiles become usable on load', () => { } ] } - await seedStore(providerRecord) + await seedStore(providerRecord, { listed: true }) await seedRunningTurn(provider) openHost() - await host.restoreReadableSessions() + await host.restoreStartupSessions() - expect(host.hasSession(SESSION)).toBe(true) + // Opened only to settle its status, so the pass closed it again; a read reopens it. + expect(host.hasSession(SESSION)).toBe(false) + await host.history({ sessionId: SESSION, direction: 'tail' }) const firstCursor = restoredJournal().cursor() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -277,21 +283,27 @@ describe('already-wedged profiles become usable on load', () => { hostId: 'local' }) openHost() - await host.restoreReadableSessions() + await host.restoreStartupSessions() + await host.history({ sessionId: SESSION, direction: 'tail' }) expect(restoredJournal().cursor()).toEqual(firstCursor) expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) } ) - it('settles restart eviction through attach when a hold arrives before the boot sweep', async () => { + it('settles restart eviction through attach when a start arrives before the boot sweep', async () => { await seedStore( wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }) ) await seedRunningTurn() openHost() - await host.hold(SESSION, 'desktop-chat:restart') + // The attach adjudicates the dead owner first, which moves the fence; like a client's ensure, + // it is retried at the fence the refusal names. + const stale = await host.attach(CALLER, hostTestAttachParams(13)) + const fence = stale.ok ? 13 : (stale.refusal.currentFence ?? 13) + expect(stale.ok || stale.refusal.code === 'agent_session_checkpoint_stale').toBe(true) + expect(stale.ok || (await host.attach(CALLER, hostTestAttachParams(fence))).ok).toBe(true) expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) @@ -402,7 +414,7 @@ describe('already-wedged profiles become usable on load', () => { ) openHost() - await host.restoreReadableSessions() + await host.restoreStartupSessions() const lease = store.getRecord(SESSION)!.lease expect(lease).toMatchObject({ handoffStage: null, unreconciled: false }) @@ -431,7 +443,7 @@ describe('already-wedged profiles become usable on load', () => { probeOwner: async () => ({ outcome: 'identity-matched', matchedOn: ['spawn-token'] }) }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'conflicted', @@ -446,7 +458,7 @@ describe('already-wedged profiles become usable on load', () => { await seedStore(wedgedRecord({ claimStatus: 'released', handoffStage: 'recovering' })) openHost() - await host.restoreReadableSessions() + await host.restoreStartupSessions() const lease = store.getRecord(SESSION)!.lease expect(lease).toMatchObject({ handoffStage: null, unreconciled: false }) @@ -479,7 +491,7 @@ describe('already-wedged profiles become usable on load', () => { } }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() const lease = store.getRecord(SESSION)!.lease expect(lease).toMatchObject({ handoffStage: null, unreconciled: false }) @@ -520,10 +532,11 @@ describe('already-wedged profiles become usable on load', () => { stopOwnerProcess: (pid) => order.push(`stop:${pid}`) }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() expect(order).toEqual([]) expect(scan).not.toHaveBeenCalled() - await host.hold(SESSION, 'holder-1') + const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? null + expect(await host.attach(CALLER, hostTestAttachParams(fence))).toMatchObject({ ok: true }) expect(order).toEqual(['acquire']) }) @@ -537,7 +550,7 @@ describe('already-wedged profiles become usable on load', () => { }) ) openHost({ probeOwner: async () => ({ outcome: 'indeterminate', reason: 'no answer' }) }) - await host.restoreReadableSessions() + await host.restoreStartupSessions() const refused = await host.attach(CALLER, hostTestAttachParams(13)) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts index 8eac02dc17b7..366ce1cf948e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts @@ -17,6 +17,7 @@ import { agentSessionProviderHandleRoot } from '../../../shared/agent-session-provider-handle' import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionResumeMarker, @@ -135,8 +136,7 @@ function liveTasks( type WorkingCandidateSession = { journal: AgentSessionJournal /** Only this host generation's own child counts. A restored-for-reading journal has none. */ - hasProviderChild: boolean - fence?: number + child: { fence: number } | null } /** The offer one session is owed, taken right before teardown stops its provider child; null when @@ -154,16 +154,24 @@ export function structuredAgentSessionWorkingAtStop(input: { }): AgentSessionResumeMarker | null { const { sessionId, session } = input // A journal this host cannot read tells us nothing about what the turn was doing. - if (!session?.hasProviderChild || session.journal.isReadOnly) { + if (!session?.child || session.journal.isReadOnly) { return null } const snapshot = session.journal.snapshot() + // A queued message reached no agent, so it is no work to resume: quit rejects it as never sent. + const handedOver = snapshot.submissions.filter( + (submission) => !isQueuedAgentJournalSubmission(submission) + ) const roster = input.backgroundTasks(sessionId) - const status = structuredAgentSessionShownStatus(snapshot, roster, session.fence) + const status = structuredAgentSessionShownStatus( + { items: snapshot.items, submissions: handedOver }, + roster, + session.child.fence + ) if (status.state === 'done') { return null } - const work = structuredAgentSessionResumeWork(snapshot.items, snapshot.submissions) + const work = structuredAgentSessionResumeWork(snapshot.items, handedOver) const head = agentSessionProviderHandleChainHead( input.getRecord(sessionId)?.providerHandleChain ?? [] ) @@ -184,6 +192,7 @@ export function structuredAgentSessionWorkingAtStop(input: { recordedAt: input.now, trigger: input.trigger, teardownId: input.teardownId, + journalCursor: snapshot.cursor, // Root, not key: the close path advances Claude's leaf moments after this runs, and a key // comparison would then refuse the session forever. providerHandleRoot: agentSessionProviderHandleRoot(head.handle), diff --git a/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts b/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts index 2199da053922..64f1ee652e78 100644 --- a/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts @@ -9,12 +9,21 @@ export async function recoverInterruptedCompaction( fence: number ): Promise { const command = store.getRecord(sessionId)?.conversationCommand - if ( - command?.command !== 'compact' || - command.phase !== 'prepared' || - command.runtimeFence === undefined || - command.runtimeFence === fence - ) { + if (command?.runtimeFence === undefined || command.runtimeFence === fence) { + return + } + await settleInterruptedCompaction(store, sessionId, journal, fence) +} + +/** A compaction still prepared whose child can no longer finish it: its outcome is unknown. */ +export async function settleInterruptedCompaction( + store: AgentSessionRecordStore, + sessionId: string, + journal: AgentSessionJournal, + fence: number +): Promise { + const command = store.getRecord(sessionId)?.conversationCommand + if (command?.command !== 'compact' || command.phase !== 'prepared') { return } const error = 'Previous compaction completion could not be confirmed after session recovery.' diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts index 78b9f4a59d0f..fe6e55019759 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts @@ -48,9 +48,12 @@ export function conversationCommandBlocked( : 'Wait for background tasks to finish before using this command.' } if ( - ctx.journal - .submissions() - .some((entry) => entry.dispatchState === 'pending' || entry.dispatchState === 'unknown') + ctx.journal.submissions().some( + (entry) => + entry.dispatchState === 'pending' || + // Doubt left by an earlier child is not this one's work in flight. + (entry.dispatchState === 'unknown' && entry.recovered !== true && entry.fence === ctx.fence) + ) ) { return 'Resolve pending or unconfirmed messages before using this command.' } diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts index 37d8d4ee83d7..e6e6d70d6223 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts @@ -117,7 +117,7 @@ describe('host conversation commands', () => { }) expect(compact).toHaveBeenCalledTimes(1) expect(adapter.dispatch).not.toHaveBeenCalled() - const history = host.history({ sessionId: HOST_TEST_SESSION, direction: 'tail' }) + const history = await host.history({ sessionId: HOST_TEST_SESSION, direction: 'tail' }) expect(history.page.submissions).toEqual([]) expect( history.page.items.some( @@ -204,7 +204,7 @@ describe('host conversation commands', () => { }) expect(store.getRecord(HOST_TEST_SESSION)).not.toBeNull() expect(store.listVisibleSessionIds()).toEqual([nextId]) - expect(host.history({ sessionId: nextId, direction: 'tail' }).page.items).toEqual([]) + expect((await host.history({ sessionId: nextId, direction: 'tail' })).page.items).toEqual([]) expect(await host.conversationCommand(caller, params)).toMatchObject({ ok: true, replayed: true, diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts index 5894defc4839..77dc2082edff 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts @@ -16,6 +16,7 @@ import { } from './structured-agent-session-attach' import { admitAndRunAgentSessionMutation } from './structured-agent-session-mutation-admission' import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' +import { openWithAgent } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionCaller } from './structured-agent-session-host-types' import type { StructuredAgentSessionHost } from './structured-agent-session-host' import { conversationCommandBlocked } from './structured-conversation-command-admission' @@ -52,6 +53,8 @@ export function runStructuredConversationCommand( adapter: context.deps.adapter, callerKey: caller.callerKey, envelope, + // Only the provider can do this, so an agent at rest is started first. + prepareSession: openWithAgent(context, params.envelope), journal: () => context.sessions.get(sessionId)?.journal, publish: (journal) => context.publish(sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, diff --git a/src/main/runtime/agent-session-operation-admission.ts b/src/main/runtime/agent-session-operation-admission.ts index e7a4165120ed..f16d48efb9d8 100644 --- a/src/main/runtime/agent-session-operation-admission.ts +++ b/src/main/runtime/agent-session-operation-admission.ts @@ -36,6 +36,7 @@ export type AgentSessionMutationOperationAdmission = { hostFingerprint: string now: number operationIdScope?: 'global' + conversationWrite?: true } export type AgentSessionMutationOperationDecision = { @@ -149,7 +150,8 @@ export function admitAgentSessionMutationOperation( envelope: args.envelope, hostFingerprint: args.hostFingerprint, ledger: ledger.decision, - lease: record.lease + lease: record.lease, + ...(args.conversationWrite ? { conversationWrite: true } : {}) }) if (ledger.decision.decision === 'admit' && admission.decision === 'refused') { ledger.rows.delete(agentSessionOperationKey(operation.callerKey, operation.operationId)) diff --git a/src/main/runtime/agent-session-record-options.ts b/src/main/runtime/agent-session-record-options.ts index 8cf88d685f37..c2477f99e2e6 100644 --- a/src/main/runtime/agent-session-record-options.ts +++ b/src/main/runtime/agent-session-record-options.ts @@ -7,7 +7,10 @@ export function replaceAgentSessionRecordOptions( record: AgentSessionRecord, replacement: AgentSessionOptionsReplacement ): AgentSessionRecord { - if (record.lease.runtimeFence !== replacement.fence || record.lease.claimStatus !== 'live') { + const { lease } = record + // At rest the host is the only writer: a pick is intent the next start replays. + const atRest = lease.claimStatus === 'released' && lease.ownerProcess === null + if (lease.runtimeFence !== replacement.fence || (lease.claimStatus !== 'live' && !atRest)) { throw new Error('agent_session_ownership_unknown') } return { ...record, options: { ...replacement.options }, updatedAt: replacement.now } diff --git a/src/main/runtime/agent-session-record-store-tab-table.test.ts b/src/main/runtime/agent-session-record-store-tab-table.test.ts index e0ecfec28358..1af97b7de172 100644 --- a/src/main/runtime/agent-session-record-store-tab-table.test.ts +++ b/src/main/runtime/agent-session-record-store-tab-table.test.ts @@ -73,7 +73,8 @@ describe('chat tab table', () => { await store.reserveOwner(reserveRequest({ surfaceTabId: 'tab-alpha' })) // A create that dies before its tab is shown leaves nothing to restore or release. expect(store.getSessionTabId('session-alpha')).toBeNull() - expect((await readFileJson()).sessionTabs).toBeUndefined() + // The table exists from the store's first open, and lists nothing. + expect((await readFileJson()).sessionTabs).toEqual([]) await store.setSessionTabVisibility('session-alpha', true, 'tab-alpha') expect(store.getSessionTabId('session-alpha')).toBe('tab-alpha') @@ -170,6 +171,94 @@ describe('chat tab table', () => { expect(persisted.records['session-alpha'].surfaceTabId).toBe('tab-alpha') }) + it('seeds a store with no index from the saved session, so one later write cannot drop the rest (L4)', async () => { + const first = await open() + for (const [index, sessionId] of ['session-alpha', 'session-beta', 'session-gamma'].entries()) { + await first.reserveOwner( + reserveRequest({ + sessionId, + operation: { + callerKey: 'client-1', + operationId: operationId(), + fingerprint: `fp-${index}` + } + }) + ) + } + // What a build before any tab index wrote: chats listed only in the saved workspace session. + const raw = await readFileJson() + delete raw.sessionTabs + delete raw.visibleSessionIds + const legacy = JSON.stringify(raw) + await writeFile(filePath(), legacy) + const saved = ['session-alpha', 'session-beta', 'session-gamma', 'session-without-record'] + + const seeded = await AgentSessionRecordStore.open({ + directory, + hostId: 'local', + savedTabSessionIds: () => saved + }) + expect(seeded.listVisibleSessionIds()).toEqual([ + 'session-alpha', + 'session-beta', + 'session-gamma' + ]) + expect(await readFile(filePath(), 'utf-8')).toBe(legacy) + + await seeded.setSessionTabVisibility('session-beta', false) + expect((await readFileJson()).visibleSessionIds).toEqual(['session-alpha', 'session-gamma']) + const restarted = await AgentSessionRecordStore.open({ + directory, + hostId: 'local', + savedTabSessionIds: () => saved + }) + expect(restarted.listVisibleSessionIds()).toEqual(['session-alpha', 'session-gamma']) + }) + + it('keeps the seed when another writer rewrites the legacy file before the first write', async () => { + const first = await open() + for (const [index, sessionId] of ['session-alpha', 'session-beta', 'session-gamma'].entries()) { + await first.reserveOwner( + reserveRequest({ + sessionId, + operation: { + callerKey: 'client-1', + operationId: operationId(), + fingerprint: `fp-${index}` + } + }) + ) + } + const raw = await readFileJson() + delete raw.sessionTabs + delete raw.visibleSessionIds + await writeFile(filePath(), JSON.stringify(raw)) + const seeded = await AgentSessionRecordStore.open({ + directory, + hostId: 'local', + savedTabSessionIds: () => ['session-alpha', 'session-beta', 'session-gamma'] + }) + // An outgoing build, which knows no tab index, still writes the file mid-restart. + raw.retiredClaimKeys = [ + ...(raw.retiredClaimKeys ?? []), + { keyId: 'key-outgoing', retiredAt: NOW } + ] + await writeFile(filePath(), JSON.stringify(raw)) + + await seeded.setSessionTabVisibility('session-beta', true) + + expect(seeded.listVisibleSessionIds()).toEqual([ + 'session-alpha', + 'session-beta', + 'session-gamma' + ]) + expect((await readFileJson()).visibleSessionIds).toEqual([ + 'session-alpha', + 'session-beta', + 'session-gamma' + ]) + }) + it('seeds a chat cleared before the upgrade under the id its tab opened with', async () => { const first = await open() for (const [index, sessionId] of ['session-alpha', 'clear-one', 'clear-two'].entries()) { diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index 445260cc47b4..8e07b86ebfab 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -70,7 +70,7 @@ import { agentSessionStorePath, type AgentSessionStoreState } from './agent-session-record-store-file' -import { setAgentSessionTabVisibility } from './agent-session-tab-table' +import { seedAgentSessionTabTable, setAgentSessionTabVisibility } from './agent-session-tab-table' import { loadProtectedAgentSessionStore } from './agent-session-record-store-security' import { AgentSessionStoreTransactionQueue, @@ -83,7 +83,12 @@ export const AGENT_SESSION_LEASE_TTL_MS = 30_000, export class AgentSessionRecordStore { private constructor(private readonly transactions: AgentSessionStoreTransactionQueue) {} - static async open(args: { directory: string; hostId: string }): Promise { + static async open(args: { + directory: string + hostId: string + /** The saved workspace session's chat tabs, for a store that predates any tab index. */ + savedTabSessionIds?: () => readonly string[] + }): Promise { const filePath = agentSessionStorePath(args.directory) const loaded = await loadProtectedAgentSessionStore(filePath, args.hostId) // Why: every persisted lease is unreconciled until this host adjudicates it, so a restart @@ -93,11 +98,18 @@ export class AgentSessionRecordStore { // write here: a rewrite at open would read as an external change to any other holder of the // file mid-restart. markAgentSessionStoreLeasesUnreconciled(loaded.state) + // After the revision for the same reason: the seeded table reaches disk with the first write. + // Re-seeded on reload too, or an outgoing build's write before that one would drop it. + const seed = (state: AgentSessionStoreState) => + seedAgentSessionTabTable(state, args.savedTabSessionIds?.() ?? []) + const seeded = seed(loaded.state) + const needsRewrite = loaded.needsRewrite || loaded.legacyHandoffLeasesNormalized || seeded const transactions = AgentSessionStoreTransactionQueue.fromLoadedStore( filePath, args.hostId, - { ...loaded, needsRewrite: loaded.needsRewrite || loaded.legacyHandoffLeasesNormalized }, - diskRevision + { ...loaded, needsRewrite }, + diskRevision, + seed ) if (loaded.needsRewrite && !loaded.readOnly && !loaded.recoveredFromBackup) { await transactions.persistLoadedRewrite() @@ -131,11 +143,6 @@ export class AgentSessionRecordStore { this.state.records.has(sessionId) ) - getVisibleSessionTabIndex = (): { present: boolean; sessionIds: string[] } => ({ - present: this.state.sessionTabs !== null, - sessionIds: this.listVisibleSessionIds() - }) - /** The id of the chat tab showing this conversation, if one does. */ getSessionTabId = (sessionId: string): string | null => this.state.sessionTabs?.tabIdFor(sessionId) ?? null diff --git a/src/main/runtime/agent-session-recovery-capsule.ts b/src/main/runtime/agent-session-recovery-capsule.ts index 8b54871b9f4a..80ed0bb726e2 100644 --- a/src/main/runtime/agent-session-recovery-capsule.ts +++ b/src/main/runtime/agent-session-recovery-capsule.ts @@ -201,15 +201,20 @@ export class AgentSessionRecoveryCapsule { /** Forgets the named sessions whatever their state. Unlike `clearAll`, this is not a fence: a * later teardown of the same chat may record a fresh offer. */ - dismiss(sessionIds: readonly string[], now: number): Promise { + dismiss( + sessionIds: readonly string[], + now: number, + /** A record this answers true for stays: read against the stored marker, under the lock. */ + keep: (marker: AgentSessionResumeMarker) => boolean = () => false + ): Promise { return withFileTransactionLock(this.filePath, async () => { const named = new Set(sessionIds) const state = await this.readState() const { entries, failed } = normalizeState(state, now) const dismissed = new Set( [...entries, ...failed] + .filter((record) => named.has(record.marker.sessionId) && !keep(record.marker)) .map((record) => record.marker.sessionId) - .filter((sessionId) => named.has(sessionId)) ) if (dismissed.size > 0) { await this.publish( diff --git a/src/main/runtime/agent-session-saved-status-store.test.ts b/src/main/runtime/agent-session-saved-status-store.test.ts new file mode 100644 index 000000000000..96d22da7f323 --- /dev/null +++ b/src/main/runtime/agent-session-saved-status-store.test.ts @@ -0,0 +1,213 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Worker } from 'node:worker_threads' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, + type StructuredAgentSessionSavedStatus +} from '../../shared/structured-agent-session-saved-status' +import Database from '../sqlite/sync-database' +import { + AGENT_SESSION_SAVED_STATUS_FILE, + AgentSessionSavedStatusStore +} from './agent-session-saved-status-store' + +let dirs: string[] = [] +let stores: AgentSessionSavedStatusStore[] = [] + +afterEach(() => { + stores.forEach((store) => store.close()) + stores = [] + dirs.forEach((dir) => rmSync(dir, { recursive: true, force: true })) + dirs = [] + vi.restoreAllMocks() +}) + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'orca-saved-status-')) + dirs.push(dir) + return dir +} + +function open(dir: string): AgentSessionSavedStatusStore { + const store = AgentSessionSavedStatusStore.open(dir) + stores.push(store) + return store +} + +function saved(sequence: number, latestPrompt = 'hello'): StructuredAgentSessionSavedStatus { + return { + v: STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, + cursor: { epoch: 'epoch-a', sequence }, + projection: { status: 'idle', latestPrompt, lastAssistantMessage: 'done' }, + lastActivityAt: 1_000 + sequence + } +} + +/** Holds the file's write lock from another thread. Once `flushing` is signalled it keeps the lock + * a further 300 ms and commits: a flush that waits on the lock succeeds, one that skips does not. */ +async function holdWriteLock(path: string) { + const signal = new Int32Array(new SharedArrayBuffer(4)) + const worker = new Worker( + `const { parentPort, workerData } = require('node:worker_threads') + const { DatabaseSync } = require('node:sqlite') + const db = new DatabaseSync(workerData.path) + db.exec('BEGIN IMMEDIATE') + parentPort.postMessage('locked') + Atomics.wait(workerData.signal, 0, 0) + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 300) + db.exec('COMMIT') + db.close()`, + { eval: true, workerData: { path, signal } } + ) + await new Promise((resolve, reject) => { + worker.once('message', () => resolve()) + worker.once('error', reject) + }) + return { + flushing: () => { + Atomics.store(signal, 0, 1) + Atomics.notify(signal, 0) + }, + exited: new Promise((resolve) => worker.once('exit', () => resolve())) + } +} + +describe('saved chat status store', () => { + it('reads back what it wrote after a reopen', () => { + const dir = tempDir() + const store = open(dir) + store.record('s1', saved(10)) + expect(store.read('s1')).toEqual(saved(10)) + store.close() + expect(open(dir).read('s1')).toEqual(saved(10)) + }) + + it('writes many coalesced entries in one transaction, and flushes at close', () => { + const dir = tempDir() + const store = open(dir) + const exec = vi.spyOn(Database.prototype, 'exec') + for (let sequence = 1; sequence <= 20; sequence += 1) { + store.record('s1', saved(sequence)) + store.record('s2', saved(sequence, 'other')) + } + expect(exec.mock.calls.filter(([sql]) => sql === 'BEGIN IMMEDIATE')).toHaveLength(0) + store.close() + expect(exec.mock.calls.filter(([sql]) => sql === 'BEGIN IMMEDIATE')).toHaveLength(1) + const reopened = open(dir) + expect(reopened.read('s1')).toEqual(saved(20)) + expect(reopened.read('s2')).toEqual(saved(20, 'other')) + }) + + it('skips a flush at once while another process holds the lock, and the next flush writes', async () => { + const dir = tempDir() + const store = open(dir) + store.record('s1', saved(10)) + const lock = await holdWriteLock(join(dir, AGENT_SESSION_SAVED_STATUS_FILE)) + lock.flushing() + // A flush that waited on the lock would outlast the holder and succeed; a skip answers false. + expect(store.flush()).toBe(false) + await lock.exited + expect(store.flush()).toBe(true) + store.close() + expect(open(dir).read('s1')).toEqual(saved(10)) + }) + + it('deletes and recreates a file that is not a database, and later writes land', () => { + const dir = tempDir() + writeFileSync(join(dir, AGENT_SESSION_SAVED_STATUS_FILE), Buffer.alloc(8192, 7)) + const store = open(dir) + expect(store.enabled).toBe(true) + store.record('s1', saved(3)) + store.close() + expect(open(dir).read('s1')).toEqual(saved(3)) + }) + + it('recreates the file when a flush finds it corrupt after SQLite already rolled back', () => { + const dir = tempDir() + const store = open(dir) + store.record('before', saved(1)) + store.flush() + const exec = Database.prototype.exec + let failCommit = true + vi.spyOn(Database.prototype, 'exec').mockImplementation(function ( + this: Database.Database, + sql: string + ) { + if (sql === 'COMMIT' && failCommit) { + failCommit = false + // SQLite ends the transaction itself on a corrupt page, before the error reaches us. + exec.call(this, 'ROLLBACK') + throw Object.assign(new Error('database disk image is malformed'), { + code: 'SQLITE_CORRUPT' + }) + } + return exec.call(this, sql) + }) + store.record('after', saved(2)) + expect(store.flush()).toBe(false) + store.close() + const reopened = open(dir) + expect(reopened.read('before')).toBeNull() + expect(reopened.read('after')).toEqual(saved(2)) + }) + + it('backs off a flush that keeps failing instead of retrying every 250 ms', () => { + vi.useFakeTimers() + try { + const store = open(tempDir()) + const exec = Database.prototype.exec + const begins = vi.spyOn(Database.prototype, 'exec').mockImplementation(function ( + this: Database.Database, + sql: string + ) { + if (sql === 'BEGIN IMMEDIATE') { + throw new Error('database or disk is full') + } + return exec.call(this, sql) + }) + const attempts = () => begins.mock.calls.filter(([sql]) => sql === 'BEGIN IMMEDIATE').length + store.record('s1', saved(1)) + vi.advanceTimersByTime(250) + expect(attempts()).toBe(1) + + store.record('s1', saved(2)) + vi.advanceTimersByTime(250) + expect(attempts()).toBe(1) + vi.advanceTimersByTime(250) + expect(attempts()).toBe(2) + } finally { + vi.useRealTimers() + } + }) + + it('leaves a newer build’s file untouched and reads or writes nothing from it', () => { + const dir = tempDir() + const path = join(dir, AGENT_SESSION_SAVED_STATUS_FILE) + const newer = new Database(path) + newer.exec('CREATE TABLE later (x INTEGER)') + newer.pragma('user_version = 99') + newer.close() + const before = readFileSync(path) + const store = open(dir) + expect(store.enabled).toBe(false) + store.record('s1', saved(3)) + expect(store.read('s1')).toBeNull() + store.close() + expect(readFileSync(path).equals(before)).toBe(true) + }) + + it('prunes entries whose chat is gone', () => { + const dir = tempDir() + const store = open(dir) + store.record('keep', saved(1)) + store.record('gone', saved(1)) + store.flush() + store.prune((sessionId) => sessionId === 'keep') + store.close() + const reopened = open(dir) + expect(reopened.read('keep')).toEqual(saved(1)) + expect(reopened.read('gone')).toBeNull() + }) +}) diff --git a/src/main/runtime/agent-session-saved-status-store.ts b/src/main/runtime/agent-session-saved-status-store.ts new file mode 100644 index 000000000000..aaff1182823f --- /dev/null +++ b/src/main/runtime/agent-session-saved-status-store.ts @@ -0,0 +1,283 @@ +// Each chat's last settled status, kept across a restart so the host can list a chat without +// replaying its journal. A cache: every entry is checked against its journal's position before it +// is used, so a lost or stale write costs one journal open and nothing else. +// +// Writes are coalesced and never on the hot path: the status memo recomputes on every journal +// move, so an entry only joins an in-memory dirty map, and a short timer writes the map in one +// transaction. A lock held by another process is a skip the next flush retries, never a wait on +// the main thread. + +import { rmSync } from 'node:fs' +import { join } from 'node:path' +import type { StructuredAgentSessionSavedStatus } from '../../shared/structured-agent-session-saved-status' +import { parseStructuredAgentSessionSavedProjection } from '../../shared/structured-agent-session-saved-status' +import { hardenSqliteDatabaseFiles } from '../sqlite/harden-database-files' +import { isUnusableSqliteDatabaseError } from '../sqlite/sqlite-read-failure' +import Database from '../sqlite/sync-database' + +export const AGENT_SESSION_SAVED_STATUS_FILE = 'agent-session-saved-status.db' +const SCHEMA_VERSION = 1 +const FLUSH_DELAY_MS = 250 +// A flush that keeps failing waits twice as long each time, up to 250 ms × 2^6 = 16 s. +const MAX_FLUSH_BACKOFF_STEPS = 6 + +const CREATE_TABLE = `CREATE TABLE saved_status ( + session_id TEXT PRIMARY KEY, + v INTEGER NOT NULL, + epoch TEXT NOT NULL, + sequence INTEGER NOT NULL, + projection_json TEXT NOT NULL, + last_activity_at INTEGER NOT NULL +)` +const SELECT_ONE = + 'SELECT v, epoch, sequence, projection_json, last_activity_at FROM saved_status WHERE session_id = ?' +const UPSERT = `INSERT INTO saved_status + (session_id, v, epoch, sequence, projection_json, last_activity_at) VALUES (?, ?, ?, ?, ?, ?) +ON CONFLICT(session_id) DO UPDATE SET v = excluded.v, epoch = excluded.epoch, + sequence = excluded.sequence, projection_json = excluded.projection_json, + last_activity_at = excluded.last_activity_at` +const SELECT_IDS = 'SELECT session_id FROM saved_status' +const DELETE_ONE = 'DELETE FROM saved_status WHERE session_id = ?' + +function warn(message: string, error: unknown): void { + console.warn(`[agent-session-saved-status] ${message}`, error) +} + +/** Null disables the store for this run: a newer build's file is left untouched. */ +function openDatabase(path: string): Database.Database | null { + const db = new Database(path) + try { + // Read before any pragma that writes: a newer build's file must stay byte-identical. + const stored = Number(db.pragma('user_version', { simple: true }) ?? 0) + if (stored > SCHEMA_VERSION) { + db.close() + return null + } + db.pragma('journal_mode = WAL') + // A lost write costs a miss, so a commit needs no fsync, and a busy lock is a skip. + db.pragma('synchronous = NORMAL') + db.pragma('busy_timeout = 0') + if (stored < SCHEMA_VERSION) { + db.exec('BEGIN IMMEDIATE') + db.exec(`DROP TABLE IF EXISTS saved_status; ${CREATE_TABLE}`) + db.pragma(`user_version = ${SCHEMA_VERSION}`) + db.exec('COMMIT') + } + hardenSqliteDatabaseFiles(path) + return db + } catch (error) { + try { + db.close() + } catch { + // The open already failed; its error is the one worth reporting. + } + throw error + } +} + +function removeDatabaseFiles(path: string): void { + for (const file of [path, `${path}-wal`, `${path}-shm`]) { + rmSync(file, { force: true }) + } +} + +export class AgentSessionSavedStatusStore { + private readonly dirty = new Map() + private timer: ReturnType | null = null + private failedFlushes = 0 + + private constructor( + private readonly path: string, + private db: Database.Database | null + ) {} + + /** Never throws: a store that cannot open runs disabled, and every chat misses. */ + static open(stateDirectory: string): AgentSessionSavedStatusStore { + const path = join(stateDirectory, AGENT_SESSION_SAVED_STATUS_FILE) + return new AgentSessionSavedStatusStore(path, AgentSessionSavedStatusStore.openOrRecreate(path)) + } + + /** A corrupt file is deleted and recreated, never left to disable the store for good. */ + private static openOrRecreate(path: string): Database.Database | null { + try { + return openDatabase(path) + } catch (error) { + if (!isUnusableSqliteDatabaseError(error)) { + warn('open failed; saved statuses are off for this run', error) + return null + } + try { + removeDatabaseFiles(path) + return openDatabase(path) + } catch (recreateError) { + warn('recreating an unusable file failed', recreateError) + return null + } + } + } + + get enabled(): boolean { + return this.db !== null + } + + read(sessionId: string): StructuredAgentSessionSavedStatus | null { + const pending = this.dirty.get(sessionId) + if (pending) { + return pending + } + try { + const row = this.db?.prepare(SELECT_ONE).get(sessionId) + const projection = + typeof row?.projection_json === 'string' + ? parseStructuredAgentSessionSavedProjection(row.projection_json) + : null + if ( + !row || + !projection || + typeof row.v !== 'number' || + typeof row.epoch !== 'string' || + typeof row.sequence !== 'number' || + typeof row.last_activity_at !== 'number' + ) { + return null + } + return { + v: row.v, + cursor: { epoch: row.epoch, sequence: row.sequence }, + projection, + lastActivityAt: row.last_activity_at + } + } catch (error) { + warn('read failed', error) + return null + } + } + + /** Coalesced: the latest entry per chat is written by the next flush. */ + record(sessionId: string, saved: StructuredAgentSessionSavedStatus): void { + if (!this.db) { + return + } + this.dirty.set(sessionId, saved) + this.timer ??= setTimeout( + () => { + this.timer = null + this.flush() + }, + FLUSH_DELAY_MS * 2 ** Math.min(this.failedFlushes, MAX_FLUSH_BACKOFF_STEPS) + ) + this.timer.unref?.() + } + + /** Writes every dirty entry in one transaction. False when the write was skipped; the entries + * stay dirty for the next flush. */ + flush(): boolean { + const db = this.db + if (!db || this.dirty.size === 0) { + return true + } + const entries = [...this.dirty] + try { + db.exec('BEGIN IMMEDIATE') + try { + const upsert = db.prepare(UPSERT) + for (const [sessionId, saved] of entries) { + upsert.run( + sessionId, + saved.v, + saved.cursor.epoch, + saved.cursor.sequence, + JSON.stringify(saved.projection), + saved.lastActivityAt + ) + } + db.exec('COMMIT') + } catch (error) { + // SQLite may have rolled back already; a second ROLLBACK would mask a corrupt file. + if (db.isTransaction) { + db.exec('ROLLBACK') + } + throw error + } + } catch (error) { + warn('flush skipped; the next flush retries', error) + this.failedFlushes += 1 + if (isUnusableSqliteDatabaseError(error)) { + this.recreate() + } + return false + } + this.failedFlushes = 0 + for (const [sessionId, saved] of entries) { + if (this.dirty.get(sessionId) === saved) { + this.dirty.delete(sessionId) + } + } + return true + } + + /** Drops entries for chats that no longer exist, so the table stays bounded by the records. */ + prune(keep: (sessionId: string) => boolean): void { + for (const sessionId of this.dirty.keys()) { + if (!keep(sessionId)) { + this.dirty.delete(sessionId) + } + } + const db = this.db + if (!db) { + return + } + try { + const stale = db + .prepare(SELECT_IDS) + .all() + .map((row) => row.session_id) + .filter((sessionId): sessionId is string => typeof sessionId === 'string') + .filter((sessionId) => !keep(sessionId)) + if (stale.length === 0) { + return + } + db.exec('BEGIN IMMEDIATE') + const remove = db.prepare(DELETE_ONE) + for (const sessionId of stale) { + remove.run(sessionId) + } + db.exec('COMMIT') + } catch (error) { + if (db.isTransaction) { + db.exec('ROLLBACK') + } + warn('prune skipped', error) + } + } + + /** Flushes, then closes. After host teardown, so teardown's own settlements are written. */ + close(): void { + if (this.timer) { + clearTimeout(this.timer) + this.timer = null + } + this.flush() + try { + this.db?.close() + } catch (error) { + warn('close failed', error) + } + this.db = null + } + + private recreate(): void { + try { + this.db?.close() + } catch { + // The file is being replaced; its handle has nothing left to lose. + } + try { + removeDatabaseFiles(this.path) + this.db = openDatabase(this.path) + } catch (error) { + warn('recreating an unusable file failed', error) + this.db = null + } + } +} diff --git a/src/main/runtime/agent-session-store-transaction-queue.ts b/src/main/runtime/agent-session-store-transaction-queue.ts index 38b65ef62498..bff4bbbb8ee7 100644 --- a/src/main/runtime/agent-session-store-transaction-queue.ts +++ b/src/main/runtime/agent-session-store-transaction-queue.ts @@ -64,7 +64,9 @@ export class AgentSessionStoreTransactionQueue { private diskStoreFound: boolean, public state: AgentSessionStoreState, private diskRevision: string, - private needsRewrite: boolean + private needsRewrite: boolean, + /** Re-applied to every reload of another writer's file; true when it changed the state. */ + private readonly normalizeLoaded: (state: AgentSessionStoreState) => boolean = () => false ) { this.diskRecoveredFromBackup = recoveredFromBackup } @@ -73,7 +75,8 @@ export class AgentSessionStoreTransactionQueue { filePath: string, hostId: string, loaded: LoadedAgentSessionStore, - diskRevision: string + diskRevision: string, + normalizeLoaded?: (state: AgentSessionStoreState) => boolean ): AgentSessionStoreTransactionQueue { return new AgentSessionStoreTransactionQueue( filePath, @@ -83,7 +86,8 @@ export class AgentSessionStoreTransactionQueue { loaded.storeFound, loaded.state, diskRevision, - loaded.needsRewrite + loaded.needsRewrite, + normalizeLoaded ) } @@ -165,9 +169,10 @@ export class AgentSessionStoreTransactionQueue { throw new Error('agent_session_legacy_required') } markLoadedLeasesUnreconciled(loaded.state) + const normalized = this.normalizeLoaded(loaded.state) this.state = loaded.state this.diskRevision = diskRevision - this.needsRewrite = loaded.needsRewrite + this.needsRewrite = loaded.needsRewrite || normalized } } diff --git a/src/main/runtime/agent-session-tab-table.ts b/src/main/runtime/agent-session-tab-table.ts index 3855d4319895..90bf876c8ef4 100644 --- a/src/main/runtime/agent-session-tab-table.ts +++ b/src/main/runtime/agent-session-tab-table.ts @@ -130,6 +130,29 @@ export function setAgentSessionTabVisibility( } } +/** + * A store written before any tab index lists its chats only in the saved workspace session. The + * table is seeded from that session at open, and again on any reload of a file still without one, + * so no later single write can create a table holding one chat and drop the rest. True when it + * seeded. + */ +export function seedAgentSessionTabTable( + state: AgentSessionStoreState, + savedTabSessionIds: readonly string[] +): boolean { + if (state.sessionTabs !== null) { + return false + } + const table = new AgentSessionTabTable() + for (const sessionId of savedTabSessionIds) { + if (state.records.has(sessionId)) { + table.show(sessionId) + } + } + state.sessionTabs = table + return true +} + export type PersistedAgentSessionTab = { tabId: string; sessionId: string } export function serializeAgentSessionTabTable(table: AgentSessionTabTable): { diff --git a/src/main/runtime/claude-structured-resumed-start-failure.test.ts b/src/main/runtime/claude-structured-resumed-start-failure.test.ts index 8fcef7e4a48b..8d379388a40b 100644 --- a/src/main/runtime/claude-structured-resumed-start-failure.test.ts +++ b/src/main/runtime/claude-structured-resumed-start-failure.test.ts @@ -38,11 +38,11 @@ describe('a reopened Claude chat whose CLI dies before initialize', () => { await waitForStructuredAgentSessionRecovery() await host.close(SESSION) - // The user reopens it; this time the CLI never answers, then dies, and its tree is unprovable. + // The user reopens it and sends; this time the CLI never answers, then dies, and its tree is + // unprovable. Opening starts nothing: the send does. claude.behave(SESSION, { initHangs: true, closeUnproven: true }) - await host.hold(SESSION, 'surface-1') const events: AgentSessionSubscribeEvent[] = [] - host.subscribe({ id: 'sub-1', sessionId: SESSION, emit: (event) => events.push(event) }) + await host.subscribe({ id: 'sub-1', sessionId: SESSION, emit: (event) => events.push(event) }) const body = hostTestMessage('hello') const fence = host.deps.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 const sent = await host.send(CALLER, { @@ -59,6 +59,8 @@ describe('a reopened Claude chat whose CLI dies before initialize', () => { body }) expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // Accepted first; the delivery loop starts the second child after. + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(2)) claude.child(SESSION).exit(new Error(DIAGNOSTIC)) await waitForStructuredAgentSessionRecovery() @@ -69,11 +71,9 @@ describe('a reopened Claude chat whose CLI dies before initialize', () => { ) ) // Never written, so it did not happen: refused, not left in doubt. - const submission = host - .journalSnapshot(SESSION) - .submissions.find( - (entry) => entry.clientMessageId === (sent.ok && sent.value.clientMessageId) - ) + const submission = (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === (sent.ok && sent.value.clientMessageId) + ) expect(submission).toMatchObject({ dispatchState: 'rejected' }) }) }) diff --git a/src/main/runtime/claude-structured-send-held-for-startup.test.ts b/src/main/runtime/claude-structured-send-held-for-startup.test.ts index 19e058110553..989c7db1b96b 100644 --- a/src/main/runtime/claude-structured-send-held-for-startup.test.ts +++ b/src/main/runtime/claude-structured-send-held-for-startup.test.ts @@ -1,8 +1,8 @@ // A Claude chat is published the moment its child spawns, before the CLI has answered initialize. -// A send in that window — into a fresh start, or into the restart a send itself asked for after -// a start that failed — is admitted and held until the child proves its start. When the CLI dies -// first, the held message is rejected with the CLI's own diagnostic, the chat shows the cause -// once, and nothing is left as a delivery nobody can confirm. Against the production runtime, +// A send in that window — into a fresh start, or into the restart the delivery loop makes for a +// send after a start that failed — is accepted and stays queued until the child proves its start. +// When the CLI dies first, the queued message is rejected with the CLI's own diagnostic, the chat +// shows the cause once, and nothing is left as a delivery nobody can confirm. Against the production runtime, // adapter, record store and host, with only the CLI process scripted. import { afterEach, describe, expect, it, vi } from 'vitest' @@ -39,7 +39,7 @@ async function send(host: StructuredAgentSessionHost, text: string): Promise (item.body.kind === 'status' ? [item.body.text] : [])) +async function statusRows(host: StructuredAgentSessionHost): Promise { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' ? [item.body.text] : [] + ) } -function submission(host: StructuredAgentSessionHost, clientMessageId: string) { - return host - .journalSnapshot(SESSION) - .submissions.find((entry) => entry.clientMessageId === clientMessageId) +async function submission(host: StructuredAgentSessionHost, clientMessageId: string) { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) } /** The CLI keeps dying at startup: the latest child exits with the diagnostic once it exists. */ @@ -82,24 +82,27 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => { ok: true }) await failLatestStart(host, 1) - expect(statusRows(host)).toEqual([expect.stringContaining('not signed in')]) + expect(await statusRows(host)).toEqual([expect.stringContaining('not signed in')]) const releasedFence = fence(host) - // The send asks for the child back and is held for its start; the CLI dies again first. + // The delivery loop asks for the child back and the message waits for its start; the CLI + // dies again first. const held = await send(host, 'hello?') - expect(claude.children(SESSION)).toHaveLength(2) - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(2)) + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + ) await failLatestStart(host, 2) // Rejected with the cause, not left in doubt; one row for this attempt names it. - await vi.waitFor(() => - expect(submission(host, held)).toMatchObject({ + await vi.waitFor(async () => + expect(await submission(host, held)).toMatchObject({ dispatchState: 'rejected', // Worded for the user: the red line under the composer shows it as it stands. reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` }) ) - expect(statusRows(host)).toEqual([ + expect(await statusRows(host)).toEqual([ expect.stringContaining('not signed in'), expect.stringMatching(/stopped before it finished starting: .*not signed in \(rig\)/) ]) @@ -111,18 +114,18 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => { // The user signs in and retries: one restart, proven, written to the CLI. claude.behave(SESSION, {}) await send(host, 'hello again') - expect(claude.children(SESSION)).toHaveLength(3) - expect(fence(host)).toBe(releasedFence + 3) + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(3)) + await vi.waitFor(() => expect(fence(host)).toBe(releasedFence + 3)) await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) await vi.waitFor(() => expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') ) - expect(statusRows(host)).toHaveLength(2) + expect(await statusRows(host)).toHaveLength(2) }) }) describe('a send while the first Claude start is still answering initialize', () => { - it('is held, and written once the CLI proves its start', async () => { + it('is queued, and written once the CLI proves its start', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() await host.attach(CALLER, claude.attachParams(SESSION, null)) @@ -130,12 +133,12 @@ describe('a send while the first Claude start is still answering initialize', () await send(host, 'hello') expect(claude.child(SESSION).calls).not.toContain('send') - // The CLI answers: startup lands and the held message is written to the proven child. + // The CLI answers: startup lands and the queued message is written to the proven child. claude.child(SESSION).answerInit() await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) expect(claude.children(SESSION)).toHaveLength(1) - expect(statusRows(host)).toEqual([]) + expect(await statusRows(host)).toEqual([]) }) it('is rejected with the diagnostic when the CLI dies first, and restarts nothing', async () => { @@ -147,14 +150,14 @@ describe('a send while the first Claude start is still answering initialize', () const held = await send(host, 'hello') await failLatestStart(host, 1) - await vi.waitFor(() => - expect(submission(host, held)).toMatchObject({ + await vi.waitFor(async () => + expect(await submission(host, held)).toMatchObject({ dispatchState: 'rejected', // Worded for the user: the red line under the composer shows it as it stands. reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` }) ) - expect(statusRows(host)).toEqual([ + expect(await statusRows(host)).toEqual([ expect.stringMatching(/stopped before it finished starting: .*not signed in \(rig\)/) ]) expect(fence(host)).toBe(startedFence + 1) diff --git a/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts index 0d4b49096379..76596ee7f647 100644 --- a/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts +++ b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts @@ -1,10 +1,9 @@ -// A send restarts a chat's Claude child and is admitted against it while it is still starting. -// When that child dies before the send's dispatch reaches the adapter, the adapter has no session -// to hold the message for, so the dispatch throws. A child that never proved its start accepted -// nothing — input is only written after initialize — so the send settles `rejected` with the -// child's own diagnostic, never as a delivery nobody can confirm, and a client that was -// subscribed the whole time receives the failure row and the rejected submission over the wire. -// Against the production runtime, adapter, record store and host, with only the CLI scripted. +// A send is accepted into a chat whose Claude child is gone, and its delivery restarts the child. +// When that child dies before it proves its start, the message was never handed to it — delivery +// waits for the start — so the send settles `rejected` with the child's own diagnostic, never as a +// delivery nobody can confirm, and a client that was subscribed the whole time receives the +// failure row and the rejected submission over the wire. Against the production runtime, adapter, +// record store and host, with only the CLI scripted. import { afterEach, describe, expect, it, vi } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope' @@ -18,6 +17,11 @@ const SESSION = 'claude-send-restart-dies-first' const CALLER = { callerKey: 'client-1' } const DIAGNOSTIC = 'claude stream-json exited (code 1): claude: not signed in (rig)' +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => unknown): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + let claude = createScriptedClaudeRuntime([SESSION]) let operations = 0 /** The dispatch state each send was answered with, before any exit settled it. */ @@ -73,37 +77,27 @@ async function send(host: StructuredAgentSessionHost, text: string): Promise (item.body.kind === 'status' ? [item.body.text] : [])) +async function statusRows(host: StructuredAgentSessionHost): Promise { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' ? [item.body.text] : [] + ) } -function submission(host: StructuredAgentSessionHost, clientMessageId: string) { - return host - .journalSnapshot(SESSION) - .submissions.find((entry) => entry.clientMessageId === clientMessageId) +async function submission(host: StructuredAgentSessionHost, clientMessageId: string) { + return (await host.journalSnapshot(SESSION)).submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) } async function failLatestStart(host: StructuredAgentSessionHost, count: number): Promise { - await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(count)) + await eventually(() => expect(claude.children(SESSION)).toHaveLength(count)) claude.child(SESSION).exit(new Error(DIAGNOSTIC)) await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => + await eventually(() => expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') ) } -/** The restarted child dies the instant the send's dispatch reaches the adapter. */ -function killChildAtDispatch(host: StructuredAgentSessionHost): void { - const adapter = host.deps.adapter - const dispatch = adapter.dispatch.bind(adapter) - vi.spyOn(adapter, 'dispatch').mockImplementationOnce((input) => { - claude.child(SESSION).exit(new Error(DIAGNOSTIC)) - return dispatch(input) - }) -} - /** Everything a subscriber received, flattened to the rows and submissions it was shown. */ function received(events: AgentSessionSubscribeEvent[]) { const statusTexts: string[] = [] @@ -129,7 +123,7 @@ function received(events: AgentSessionSubscribeEvent[]) { return { statusTexts, submissions, fences } } -describe('a send whose restarted Claude child dies before the dispatch reaches the adapter', () => { +describe('a send whose restarted Claude child dies before it proves its start', () => { it('settles rejected with the diagnostic, keeps one failure row, and a Retry is one new attempt', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() @@ -139,25 +133,20 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t await failLatestStart(host, 1) const releasedFence = fence(host) - killChildAtDispatch(host) const sent = await send(host, 'hello?') - // The send's own answer already says it was not delivered; it does not wait for the exit. - expect(answered.get(sent)).toBe('rejected') - expect(claude.children(SESSION)).toHaveLength(2) - await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - ) + // Accepted: the answer comes before the restart it needs. + expect(answered.get(sent)).toBe('pending') + await failLatestStart(host, 2) // Provably not delivered, with the cause; not "unconfirmed". - await vi.waitFor(() => - expect(submission(host, sent)).toMatchObject({ + await eventually(async () => + expect(await submission(host, sent)).toMatchObject({ dispatchState: 'rejected', // Worded for the user: the red line under the composer shows it as it stands. reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` }) ) - expect(statusRows(host)).toEqual([ + expect(await statusRows(host)).toEqual([ expect.stringContaining('not signed in'), expect.stringMatching(/stopped before it finished starting: .*not signed in \(rig\)/) ]) @@ -168,40 +157,14 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t // Retry under a new id: one restart, and once the CLI is healthy the message is written. claude.behave(SESSION, {}) await send(host, 'hello again') - expect(claude.children(SESSION)).toHaveLength(3) - await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + await eventually(() => expect(claude.children(SESSION)).toHaveLength(3)) + await eventually(() => expect(claude.child(SESSION).calls).toContain('send')) expect(claude.child(SESSION).calls.filter((call) => call === 'send')).toHaveLength(1) - expect(statusRows(host)).toHaveLength(2) - }) - - it('names the diagnostic even when the exit was fully processed before the dispatch arrived', async () => { - claude.behave(SESSION, { initHangs: true }) - const host = await claude.install() - await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ - ok: true - }) - await failLatestStart(host, 1) - const adapter = host.deps.adapter - const dispatch = adapter.dispatch.bind(adapter) - vi.spyOn(adapter, 'dispatch').mockImplementationOnce(async (input) => { - claude.child(SESSION).exit(new Error(DIAGNOSTIC)) - // The adapter settles and publishes the exit; the host's own settlement waits behind this send. - await new Promise((resolve) => setTimeout(resolve, 300)) - return dispatch(input) - }) - - const sent = await send(host, 'hello?') - expect(answered.get(sent)).toBe('rejected') - await waitForStructuredAgentSessionRecovery() - expect(submission(host, sent)).toMatchObject({ - dispatchState: 'rejected', - reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` - }) - expect(statusRows(host)).toHaveLength(2) + expect(await statusRows(host)).toHaveLength(2) }) // A restart refused because its child died before it was handed over leaves one row, from the - // send, in the words any failed start uses. + // delivery, in the words any failed start uses, and rejects the message with them. it.each(['spawn', 'start-time-read'] as const)( 'leaves one row for a restart whose child exits at %s', async (at) => { @@ -213,13 +176,16 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t await failLatestStart(host, 1) claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: DIAGNOSTIC, at } }) - await expect(attempt(host, 'hello?')).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + const sent = await send(host, 'hello?') + await eventually(async () => + expect(await submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + }) + ) await waitForStructuredAgentSessionRecovery() - expect(statusRows(host)).toEqual([ + expect(await statusRows(host)).toEqual([ `The provider stopped before it finished starting: ${DIAGNOSTIC}.`, `The provider stopped before it finished starting: ${DIAGNOSTIC}.` ]) @@ -233,7 +199,7 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t ok: true }) const events: AgentSessionSubscribeEvent[] = [] - const unsubscribe = host.subscribe({ + const unsubscribe = await host.subscribe({ id: 'pane', sessionId: SESSION, emit: (event) => { @@ -242,14 +208,10 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t }) try { await failLatestStart(host, 1) - killChildAtDispatch(host) const sent = await send(host, 'hello?') - await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - ) + await failLatestStart(host, 2) - await vi.waitFor(() => { + await eventually(() => { const seen = received(events) expect(seen.submissions.get(sent)).toBe('rejected') expect(seen.statusTexts).toContainEqual( @@ -285,14 +247,14 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o return rows } - it('shows one row naming the cause per failed attempt, admitted or refused, and none once the CLI is fixed', async () => { + it('shows one row naming the cause per failed attempt, however the start died, and none once the CLI is fixed', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() const events: AgentSessionSubscribeEvent[] = [] await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ ok: true }) - const unsubscribe = host.subscribe({ + const unsubscribe = await host.subscribe({ id: 'pane', sessionId: SESSION, emit: (event) => { @@ -301,37 +263,38 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o }) try { await failLatestStart(host, 1) - await vi.waitFor(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE])) + await eventually(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE])) - // Send: admitted against the restarted child, which dies before starting. - killChildAtDispatch(host) - const sent = await attempt(host, 'hello?') - await waitForStructuredAgentSessionRecovery() - expect(sent).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'rejected', reason: STARTUP_FAILURE } } - }) - await vi.waitFor(() => + // Send: accepted, and its delivery restarts the child, which dies before starting. + const sent = await send(host, 'hello?') + await failLatestStart(host, 2) + await eventually(async () => + expect(await submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + reason: STARTUP_FAILURE + }) + ) + await eventually(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE, STARTUP_FAILURE]) ) - // Retry while still broken: this restart dies before its child is handed over, so the send - // is refused before admission. Still one row, saying the same thing. + // Retry while still broken: this restart dies before its child is handed over, so the + // delivery's start is refused. Still one row, saying the same thing, on the rejected message. claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: 'claude stream-json exited (code 1): claude: not signed in (rig)', at: 'start-time-read' } }) - await expect(attempt(host, 'hello?')).resolves.toMatchObject({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: expect.stringMatching(/couldn't restart: .*not signed in \(rig\)/) - } - }) + const retried = await send(host, 'hello?') + await eventually(async () => + expect(await submission(host, retried)).toMatchObject({ + dispatchState: 'rejected', + reason: STARTUP_FAILURE + }) + ) await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => + await eventually(() => expect([...shownRows(events).values()]).toEqual([ STARTUP_FAILURE, STARTUP_FAILURE, @@ -342,7 +305,7 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o // The CLI is fixed: Retry delivers and adds no row. claude.behave(SESSION, {}) await expect(attempt(host, 'hello?')).resolves.toMatchObject({ ok: true }) - await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + await eventually(() => expect(claude.child(SESSION).calls).toContain('send')) expect(shownRows(events).size).toBe(3) } finally { unsubscribe() diff --git a/src/main/runtime/claude-structured-session-integration.test.ts b/src/main/runtime/claude-structured-session-integration.test.ts index 6c2555e9c762..9b82bf816c1a 100644 --- a/src/main/runtime/claude-structured-session-integration.test.ts +++ b/src/main/runtime/claude-structured-session-integration.test.ts @@ -453,7 +453,7 @@ describe('a structured Claude session over agentSession.*', () => { }) it.each(['unverifiable', 'live'] as const)( - 'reopens a chat whose stop saw the Claude root exit with its tree %s', + 'restarts a chat whose stop saw the Claude root exit with its tree %s', async (tree) => { await ok('agentSession.create', createIntentParams()) const first = claude.live() @@ -463,14 +463,19 @@ describe('a structured Claude session over agentSession.*', () => { return false } const host = getStructuredAgentSessionHost() - // The idle release clock's eviction: the lease follows the root, so the host lets go. + // The lease follows the root, so the host lets go. await host?.close(SESSION) expect(host?.hasSession(SESSION)).toBe(false) - // What the chat surface's `agentSession.hold` does when the user comes back to it. - await host?.hold(SESSION, 'desktop-chat:reopen') + // The user comes back and sends: that send is what starts Claude again. + const body = { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'back' }] } + await ok('agentSession.send', { + envelope: envelope('agentSession.send', { body }, leaseOf(SESSION).runtimeFence), + body + }) - expect(claude.connections).toHaveLength(2) + await vi.waitFor(() => expect(claude.connections).toHaveLength(2)) + await vi.waitFor(() => expect(claude.live().sent).toHaveLength(1)) expect(host?.hasSession(SESSION)).toBe(true) } ) @@ -490,11 +495,9 @@ describe('a structured Claude session over agentSession.*', () => { // A descendant seen alive is one that survived the close ladder, such as an MCP server that // ignores SIGTERM; it no longer holds the chat. it.each(['unverifiable', 'live'] as const)( - 'restarts an open chat after a Claude crash whose tree was %s', + 'restarts a chat on its next send after a Claude crash whose tree was %s', async (tree) => { const created = await ok<{ fence: number }>('agentSession.create', createIntentParams()) - // The open chat surface is what asks the host to bring Claude back. - await getStructuredAgentSessionHost()?.hold(SESSION, 'desktop-chat:open') const connection = claude.live() connection.exitVerdict = { root: 'exited', tree } connection.close = async () => { @@ -517,19 +520,19 @@ describe('a structured Claude session over agentSession.*', () => { dispatchState: 'unknown', reason: 'provider_exited_before_acknowledgement' }) - // Held back, sends failed with the crash until the idle clock stopped the chat. + // The crash releases the lease; nothing restarts Claude until the chat has work for it. await waitForStructuredAgentSessionRecovery() - expect(claude.connections).toHaveLength(2) - expect(claude.live().launch.options).toMatchObject({ resume: PROVIDER_SESSION }) - const lease = leaseOf(SESSION) - expect(lease).toMatchObject({ claimStatus: 'live', handoffStage: null }) + expect(claude.connections).toHaveLength(1) + expect(leaseOf(SESSION)).toMatchObject({ claimStatus: 'released', handoffStage: null }) const next = { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after' }] } - const sent = await ok<{ submission: { dispatchState: string } }>('agentSession.send', { - envelope: envelope('agentSession.send', { body: next }, lease.runtimeFence), + await ok('agentSession.send', { + envelope: envelope('agentSession.send', { body: next }, leaseOf(SESSION).runtimeFence), body: next }) - expect(sent.submission.dispatchState).toBe('accepted') - expect(claude.live().sent).toHaveLength(1) + await vi.waitFor(() => expect(claude.connections).toHaveLength(2)) + expect(claude.live().launch.options).toMatchObject({ resume: PROVIDER_SESSION }) + await vi.waitFor(() => expect(claude.live().sent).toHaveLength(1)) + expect(leaseOf(SESSION)).toMatchObject({ claimStatus: 'live', handoffStage: null }) } ) diff --git a/src/main/runtime/claude-structured-startup-unanswered-control-request.test.ts b/src/main/runtime/claude-structured-startup-unanswered-control-request.test.ts index 38d0de314d26..ce26647ae85f 100644 --- a/src/main/runtime/claude-structured-startup-unanswered-control-request.test.ts +++ b/src/main/runtime/claude-structured-startup-unanswered-control-request.test.ts @@ -72,10 +72,10 @@ function turnReportsModel(model: string): void { }) } -function statusRows(host: StructuredAgentSessionHost): string[] { - return host - .journalSnapshot(SESSION) - .items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) +async function statusRows(host: StructuredAgentSessionHost): Promise { + return (await host.journalSnapshot(SESSION)).items.flatMap((item) => + item.body.kind === 'status' ? [item.body.text] : [] + ) } async function send(host: StructuredAgentSessionHost, text: string): Promise { @@ -119,7 +119,7 @@ describe('a Claude start whose CLI answers initialize but not a control request' timeout: DEADLINE_MS * 40 }) expect(host.deps.adapter.readOptionRestoreFailures?.(SESSION)).toEqual([]) - expect(statusRows(host)).toEqual([]) + expect(await statusRows(host)).toEqual([]) expect(claude.children(SESSION)).toHaveLength(1) // The proven child takes the next message. @@ -245,7 +245,7 @@ describe('a Claude start whose CLI answers initialize but not a control request' timeout: DEADLINE_MS * 40 }) expect(record(host)?.lease.claimStatus).toBe('live') - expect(statusRows(host)).toEqual([]) + expect(await statusRows(host)).toEqual([]) expect(claude.children(SESSION)).toHaveLength(1) }) }) diff --git a/src/main/runtime/orca-runtime-adopt-terminal-orphans-from-inventory.ts b/src/main/runtime/orca-runtime-adopt-terminal-orphans-from-inventory.ts index 5dd2908413b3..3bc5b570704a 100644 --- a/src/main/runtime/orca-runtime-adopt-terminal-orphans-from-inventory.ts +++ b/src/main/runtime/orca-runtime-adopt-terminal-orphans-from-inventory.ts @@ -4,10 +4,7 @@ import { spawnSurfaceClaimSequence, SURFACE_CLAIM_WITHOUT_STANDING } from './pty-recorded-surface-topology' -import { - observeStructuredWorker, - resolveStructuredWorkerAuthority -} from './structured-worker-authority' +import { resolveStructuredWorkerAuthority } from './structured-worker-authority' import type { RuntimeLeafRecord } from './runtime-terminal-state-records' import { OrcaRuntimeWithSubscribeToTerminalResize } from './orca-runtime-subscribe-to-terminal-resize' import type { @@ -210,12 +207,10 @@ export class OrcaRuntimeWithAdoptTerminalOrphansFromInventory extends OrcaRuntim this.getOrchestrationDbIfAvailable?.() ?? null ) if (structured) { - // `resolveBareOrchestrationRecipient` routes direct mail through this, not through - // getTerminalPaneKey. The connected-gate below exists so mail is never routed to a corpse, - // so the structured answer needs a real liveness proof too, not just a registry hit. - return observeStructuredWorker(structured.identity).status === 'live' - ? structured.identity.paneKey - : null + // `resolveBareOrchestrationRecipient` routes direct mail through this. A structured worker is + // a recipient while this runtime owns it, running or at rest: mail to one at rest is a send, + // and the send starts its agent. + return structured.identity.paneKey } const runtimePty = this.getLivePtyForHandle(handle) if (runtimePty) { diff --git a/src/main/runtime/orca-runtime-close-mobile-session-tab.ts b/src/main/runtime/orca-runtime-close-mobile-session-tab.ts index 2ef5b77069d6..032884769095 100644 --- a/src/main/runtime/orca-runtime-close-mobile-session-tab.ts +++ b/src/main/runtime/orca-runtime-close-mobile-session-tab.ts @@ -6,7 +6,6 @@ import type { } from '../../shared/runtime-types' import type { RuntimePtyTabCloseAuthority } from './runtime-terminal-state-records' import { - adjudicateAbsentMobileSessionTabClose, resolveMobileSessionLifecycleCloseContext, type MobileSessionLifecycleCloseHost } from './mobile-session-lifecycle-close-adjudication' @@ -86,10 +85,11 @@ export class OrcaRuntimeWithCloseMobileSessionTab extends OrcaRuntimeWithRefuseU observedPtyIds }) if (!snapshot || !tab) { - return adjudicateAbsentMobileSessionTabClose({ + return this.adjudicateAbsentSessionTabClose({ host: this.getMobileSessionLifecycleCloseHost(), context: lifecycleClose, worktreeId, + tabId, snapshot, reason: options.reason, addressedByPtyCloseAuthority: options.expectedPtyCloseAuthority !== undefined diff --git a/src/main/runtime/orca-runtime-close-structured-agent-session-tab.ts b/src/main/runtime/orca-runtime-close-structured-agent-session-tab.ts index 568083bf177f..7e3666220330 100644 --- a/src/main/runtime/orca-runtime-close-structured-agent-session-tab.ts +++ b/src/main/runtime/orca-runtime-close-structured-agent-session-tab.ts @@ -1,5 +1,10 @@ // @ts-nocheck -- mechanically split from OrcaRuntimeService; behavior is covered by AST equivalence and characterization tests. import { OrcaRuntimeWithCloseMobileSessionTab } from './orca-runtime-close-mobile-session-tab' +import { adjudicateAbsentMobileSessionTabClose } from './mobile-session-lifecycle-close-adjudication' +import { + committedMobileSessionTabClose, + type MobileSessionTabCloseOutcome +} from './mobile-session-tab-close-outcome' import type { RuntimeMobileSessionAgentTab, RuntimeMobileSessionBrowserTab, @@ -17,7 +22,7 @@ import { retireStructuredAgentSessionTabFrom } from './structured-agent-session- export class OrcaRuntimeWithCloseStructuredAgentSessionTab extends OrcaRuntimeWithCloseMobileSessionTab { protected async closeStructuredAgentSessionTab(tab: RuntimeMobileSessionAgentTab): Promise { - const host = getStructuredAgentSessionHost() + const host = await this.structuredAgentSessionHostForTabClose() if (host) { if (typeof host.setSessionTabVisibility === 'function') { await host.setSessionTabVisibility(tab.sessionId, false) @@ -30,6 +35,51 @@ export class OrcaRuntimeWithCloseStructuredAgentSessionTab extends OrcaRuntimeWi } } + /** A close whose tab the runtime snapshot lacks. A restored chat the renderer shows from its saved + * session before the host has published it is still real: its durable record says so, and + * refusing it would leave the next listing to revive the tab the user closed. */ + protected async adjudicateAbsentSessionTabClose( + args: Parameters[0] & { tabId: string } + ): Promise { + const userClose = args.reason === undefined || args.reason === 'user' + if ( + userClose && + !args.addressedByPtyCloseAuthority && + args.tabId.startsWith('agent-session:') + ) { + const sessionId = args.tabId.slice('agent-session:'.length) + const host = await this.structuredAgentSessionHostForTabClose() + const record = host?.deps?.store?.getRecord?.(sessionId) + // Only a chat the index still lists, in the workspace the close names; anything else is unknown. + if ( + record?.location?.workspaceId === args.worktreeId && + host.listVisibleSessionIds?.().includes(sessionId) === true + ) { + await this.closeStructuredAgentSessionTab({ + type: 'agent-session', + id: args.tabId, + sessionId + }) + return committedMobileSessionTabClose(this.clientSessionTabSelections, args.worktreeId, [ + args.tabId + ]) + } + } + return adjudicateAbsentMobileSessionTabClose(args) + } + + /** A close before startup builds the host still owes the durable index its removal, or the tab + * returns at every launch. Bookkeeping only: a failed install must not keep the tab open. */ + private async structuredAgentSessionHostForTabClose() { + if (getStructuredAgentSessionHost() || !this.hasPersistedStructuredAgentSessionStore()) { + return getStructuredAgentSessionHost() + } + await this.ensureStructuredAgentSessionHost().catch((error) => { + console.warn('[structured-agent-session] host install failed before a tab close', error) + }) + return getStructuredAgentSessionHost() + } + /** * Prunes a structured session's chat tab from whichever worktree snapshot still carries it. * diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index f2143b9772c0..f20b3c732d7c 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -19,6 +19,7 @@ import { import { resolveStructuredLaunchSeedOptions } from '../../shared/native-chat-session-option-defaults' import { hasPersistedStructuredAgentSessionStore as hasPersistedStructuredAgentSessionStoreOnDisk } from './structured-agent-session-runtime' import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' +import { structuredAgentSessionStartupPriority } from './saved-structured-agent-session-restoration' import { parseWslUncPath } from '../../shared/wsl-paths' import { parseWorkspaceKey } from '../../shared/workspace-scope' @@ -256,8 +257,17 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR } // Durable agent records must exist before daemon inventory can be reconciled against them. await this.ensureStructuredAgentSessionHost() + const host = getStructuredAgentSessionHost() + // Not awaited: the pass lists status first and opens journals behind every reader. + void host + ?.restoreStartupSessions?.( + structuredAgentSessionStartupPriority( + this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + ) + ) + .catch((error) => console.error('[structured-agent-session] startup pass failed', error)) await this.refreshMobileSessionPtyRecords() - await getStructuredAgentSessionHost()?.reconcileRestartLeases() + await host?.reconcileRestartLeases() } protected hasPersistedStructuredAgentSessionStore(): boolean { diff --git a/src/main/runtime/orca-runtime-get-worktree-ps.ts b/src/main/runtime/orca-runtime-get-worktree-ps.ts index cf7e8e56db1b..e84a0cec7f36 100644 --- a/src/main/runtime/orca-runtime-get-worktree-ps.ts +++ b/src/main/runtime/orca-runtime-get-worktree-ps.ts @@ -18,7 +18,9 @@ import { maybeAutoRenameWorkspaceOnFirstStructuredTurn } from '../agent-hooks/fi import { firstWorkRenameDeps } from '../agent-hooks/first-work-rename-runtime' import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' +import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { buildWorktreeListingPage } from './worktree-listing-host-scope' +import { structuredWorkerOwesWork } from './structured-worker-custody' import { resolveTuiAgentLaunchEnv } from '../../shared/tui-agent-launch-defaults' import { nativeChatShellEnvironmentPolicy } from '../../shared/native-chat-shell-environment' import { claudeStructuredPermissionModeForSettings } from '../claude/claude-structured-permission-mode' @@ -140,6 +142,10 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVis await installStructuredAgentSessionHost({ stateDirectory: getProfileUserDataPath(), hostId: LOCAL_EXECUTION_HOST_ID, + savedTabSessionIds: () => + collectSavedStructuredAgentSessionIds( + this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + ), claimKeyId: this.agentSessionClaimSigner.keyId, // Resolves folder workspaces as well as git worktrees, so a chat session // in a plain folder lands in the folder rather than failing to resolve. @@ -171,7 +177,11 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVis firstWorkRenameDeps(this.requireStore(), this) ) }, - ...(this.structuredAgentStatusSinkFn ? { statusSink: this.structuredAgentStatusSinkFn } : {}) + ...(this.structuredAgentStatusSinkFn ? { statusSink: this.structuredAgentStatusSinkFn } : {}), + // Read per sweep tick from the orchestration database: a worker whose dispatch is open keeps + // its agent running. No database answers no. + hasOpenDispatch: (record) => + structuredWorkerOwesWork(this.getOrchestrationDbIfAvailable?.() ?? null, record) }) } } diff --git a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts index 50b1f52d238a..75005726c2db 100644 --- a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts +++ b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts @@ -216,9 +216,10 @@ export class OrcaRuntimeWithPruneMobileSessionTabGroupLayout extends OrcaRuntime } // Why: group address resolution (Section 4.5) queries per-handle status and must not throw on stale handles; return null on any error. - getAgentStatusForHandle(handle: string): string | null { + async getAgentStatusForHandle(handle: string): Promise { // A structured worker has no pane and no title, so every PTY probe below answers null and - // `@idle` would enumerate it and then silently drop it. Its status is the journal's. + // `@idle` would enumerate it and then silently drop it. Its status is the journal's, read + // through a conversation the idle sweep may have closed. const structured = resolveStructuredWorkerAuthority(handle, this._orchestrationDb) if (structured) { return structuredWorkerAgentStatus(structured.identity.sessionId) diff --git a/src/main/runtime/orca-runtime-resolve-terminal-pane.ts b/src/main/runtime/orca-runtime-resolve-terminal-pane.ts index 1093319783a6..60047604e438 100644 --- a/src/main/runtime/orca-runtime-resolve-terminal-pane.ts +++ b/src/main/runtime/orca-runtime-resolve-terminal-pane.ts @@ -184,7 +184,7 @@ export class OrcaRuntimeWithResolveTerminalPane extends OrcaRuntimeWithGetTermin ): Promise { // Before the PTY lookup, because a structured worker has no PTY and no leaf: without this the // only peer read verb answers `terminal_handle_stale` for a perfectly live worker. - const structured = readStructuredWorkerTerminal({ + const structured = await readStructuredWorkerTerminal({ handle, db: this.getOrchestrationDbIfAvailable?.() ?? null, ...(opts.cursor === undefined ? {} : { cursor: opts.cursor }), diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index f22a09bac5e4..7d56de23d6e9 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -1,17 +1,17 @@ // @ts-nocheck -- mechanically split from OrcaRuntimeService; behavior is covered by AST equivalence and characterization tests. -import { defaultAgentChatLabel } from '../../shared/agent-session-chat-label' import { OrcaRuntimeWithGetStructuredAgentSessionCreateSupport } from './orca-runtime-get-structured-agent-session-create-support' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' -import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' -import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { - RuntimeMobileSessionAgentTab, RuntimeMobileSessionTabsSnapshot, RuntimeRepoSearchRefs } from '../../shared/runtime-types' -import { getHeadlessMobileSessionGroupId } from './mobile-session-layout-projection' +import { + appendStructuredAgentSessionTabs, + structuredAgentSessionSnapshotTabId, + type StructuredAgentSessionTabToAppend +} from './structured-agent-session-tab-append' import { DEFAULT_REPO_SEARCH_REFS_LIMIT } from './orca-runtime-postlude' import type { Repo } from '../../shared/repo-types' import type { GitAdmissionTier } from '../git/command-runner/git-exec-options' @@ -47,19 +47,16 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu } } + // Tab existence needs only the opened store: reconcile, recovery and every journal open run + // behind the answer, in the startup pass prepare kicks. protected async restoreStructuredAgentSessionTabsOnce(): Promise { - await this.prepareStructuredAgentSessionStartupRestoration() + if (this.hasPersistedStructuredAgentSessionStore()) { + await this.ensureStructuredAgentSessionHost() + void this.prepareStructuredAgentSessionStartupRestoration().catch((error) => { + console.error('[structured-agent-session] startup restoration failed', error) + }) + } const host = getStructuredAgentSessionHost() - const persistedVisibleIndex = - typeof host?.getPersistedVisibleSessionTabIndex === 'function' - ? host.getPersistedVisibleSessionTabIndex() - : { present: false, sessionIds: [] } - const profileIds = collectSavedStructuredAgentSessionIds( - this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null - ) - await host?.restoreReadableSessions( - persistedVisibleIndex.present ? persistedVisibleIndex.sessionIds : profileIds - ) for (const worktreeId of this.getKnownWorkspaceSessionWorktreeIds()) { this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { allowAttachedWindow: true, @@ -70,21 +67,41 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const replacement of host?.conversationReplacements?.() ?? []) { await this.replaceStructuredAgentSessionTab(replacement) } - for (const session of host?.listSessionTabs() ?? []) { - if (session.agent !== 'codex' && session.agent !== 'claude') { + this.publishListedStructuredAgentSessionTabs(host) + } + + /** Every listed chat in one store write per workspace, from the index and the records alone. + * Synchronous from the index read to the last store, so a close lands wholly before or after. */ + private publishListedStructuredAgentSessionTabs(host): void { + if ( + typeof host?.listVisibleSessionIds !== 'function' || + typeof host.listPersistedSessionTabs !== 'function' + ) { + return + } + const byWorkspace = new Map() + for (const tab of host.listPersistedSessionTabs(host.listVisibleSessionIds())) { + if (tab.agent !== 'codex' && tab.agent !== 'claude') { continue } - let sessionId = session.sessionId + let sessionId = tab.sessionId while (sessionId.startsWith('agent-session:')) { sessionId = sessionId.slice('agent-session:'.length) } - await this.publishStructuredAgentSessionTab({ - ...session, - agent: session.agent, - sessionId, - activate: false, - notify: false - }) + const tabs = byWorkspace.get(tab.workspaceId) ?? [] + tabs.push({ sessionId, agent: tab.agent }) + byWorkspace.set(tab.workspaceId, tabs) + } + for (const [workspaceId, tabs] of byWorkspace) { + const next = appendStructuredAgentSessionTabs( + this.mobileSessionTabsByWorktree.get(workspaceId), + workspaceId, + tabs, + { activate: false } + ) + if (next) { + this.storeMobileSessionSnapshot(workspaceId, next) + } } } @@ -107,7 +124,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu ) } const existing = this.mobileSessionTabsByWorktree.get(input.workspaceId) - const id = `agent-session:${input.sessionId}` + const id = structuredAgentSessionSnapshotTabId(input.sessionId) if (existing?.tabs.some((tab) => tab.id === id)) { // A background re-publish is a no-op — no store write, no emit — so it cannot re-surface a // client whose mirror lost the tab; healing one needs `activate` or an explicit republish. @@ -137,49 +154,20 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu } return } - const tab: RuntimeMobileSessionAgentTab = { - type: 'agent-session', - id, - title: defaultAgentChatLabel(input.agent), - sessionId: input.sessionId, - ...(input.replacesSessionId ? { replacesSessionId: input.replacesSessionId } : {}), - agent: input.agent, - isActive: input.activate - } - const tabs = [...(existing?.tabs ?? [])].map((candidate) => ({ - ...candidate, - isActive: input.activate ? false : candidate.isActive - })) - tabs.push(tab) - const priorGroups = existing?.tabGroups ?? [ - { - id: getHeadlessMobileSessionGroupId(input.workspaceId), - activeTabId: existing?.activeTabId ?? null, - tabOrder: [] - } - ] - const groupId = priorGroups.some((group) => group.id === existing?.activeGroupId) - ? existing!.activeGroupId! - : priorGroups[0]!.id - const tabGroups = priorGroups.map((group) => - group.id === groupId - ? { - ...group, - activeTabId: input.activate ? id : group.activeTabId, - tabOrder: [...group.tabOrder, id] - } - : group + const snapshot = appendStructuredAgentSessionTabs( + existing, + input.workspaceId, + [ + { + sessionId: input.sessionId, + agent: input.agent, + ...(input.replacesSessionId ? { replacesSessionId: input.replacesSessionId } : {}) + } + ], + { activate: input.activate } ) - const snapshot: RuntimeMobileSessionTabsSnapshot = { - worktree: input.workspaceId, - publicationEpoch: existing?.publicationEpoch ?? `structured:${Date.now().toString(36)}`, - snapshotVersion: (existing?.snapshotVersion ?? 0) + 1, - activeGroupId: input.activate ? groupId : (existing?.activeGroupId ?? groupId), - activeTabId: input.activate ? id : (existing?.activeTabId ?? null), - activeTabType: input.activate ? 'agent-session' : (existing?.activeTabType ?? null), - tabGroups, - ...(existing?.tabGroupLayout ? { tabGroupLayout: existing.tabGroupLayout } : {}), - tabs + if (!snapshot) { + return } const stored = this.storeMobileSessionSnapshot(input.workspaceId, snapshot) if (input.notify !== false) { diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index d6ec1e22782c..638bfbeece0b 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -4,6 +4,46 @@ import { OrcaRuntimeService } from './orca-runtime' afterEach(() => setStructuredAgentSessionHost(null)) +type RestoreInternals = { + store: { getWorkspaceSession: () => unknown } + hasPersistedStructuredAgentSessionStore(): boolean + getKnownWorkspaceSessionWorktreeIds(): Set + hydrateHeadlessMobileSessionTabsFromWorkspaceSession( + worktreeId?: string, + options?: { allowAttachedWindow?: boolean; onlyRuntimeOwnedTerminals?: boolean } + ): Set + refreshMobileSessionPtyRecords(): Promise | null> + ensureStructuredAgentSessionHost(): Promise + storeMobileSessionSnapshot(worktreeId: string, snapshot: unknown): unknown +} + +function savedSessionWithChat(sessionId: string) { + return { + activeRepoId: null, + activeWorktreeId: 'workspace-1', + activeTabId: `agent-session:${sessionId}`, + tabsByWorktree: {}, + terminalLayoutsByTabId: {}, + activeTabIdByWorktree: { 'workspace-1': `agent-session:${sessionId}` }, + unifiedTabs: { + 'workspace-1': [ + { + id: `agent-session:${sessionId}`, + entityId: sessionId, + groupId: 'group-1', + worktreeId: 'workspace-1', + contentType: 'agent-session', + label: 'Codex Chat', + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + } + } +} + describe('structured session cold restoration', () => { it('skips every heavy recovery step when no durable session store exists', async () => { const runtime = new OrcaRuntimeService() @@ -18,6 +58,7 @@ describe('structured session cold restoration', () => { internal.hasPersistedStructuredAgentSessionStore = () => false internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test reads only the host members stubbed here. setStructuredAgentSessionHost({ reconcileRestartLeases } as never) await runtime.prepareStructuredAgentSessionStartupRestoration() @@ -27,179 +68,102 @@ describe('structured session cold restoration', () => { expect(reconcileRestartLeases).not.toHaveBeenCalled() }) - it('keeps historical journal parsing outside the terminal-safety fence', async () => { + it('kicks the startup pass without waiting on it, then reconciles once', async () => { const runtime = new OrcaRuntimeService() const refresh = vi.fn(async () => new Set()) const ensureHost = vi.fn(async () => undefined) const reconcileRestartLeases = vi.fn(async () => undefined) - const restoreReadableSessions = vi.fn(async () => undefined) - const internal = runtime as unknown as { - hasPersistedStructuredAgentSessionStore(): boolean - refreshMobileSessionPtyRecords(): Promise | null> - ensureStructuredAgentSessionHost(): Promise - } + const restoreStartupSessions = vi.fn(() => new Promise(() => undefined)) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as RestoreInternals internal.hasPersistedStructuredAgentSessionStore = () => true internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ reconcileRestartLeases, restoreReadableSessions } as never) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test reads only the host members stubbed here. + setStructuredAgentSessionHost({ reconcileRestartLeases, restoreStartupSessions } as never) + await runtime.prepareStructuredAgentSessionStartupRestoration() await runtime.prepareStructuredAgentSessionStartupRestoration() expect(ensureHost).toHaveBeenCalledOnce() expect(refresh).toHaveBeenCalledOnce() expect(reconcileRestartLeases).toHaveBeenCalledOnce() - expect(restoreReadableSessions).not.toHaveBeenCalled() + expect(restoreStartupSessions).toHaveBeenCalledOnce() + expect(ensureHost.mock.invocationCallOrder[0]).toBeLessThan( + restoreStartupSessions.mock.invocationCallOrder[0] ?? Infinity + ) }) - it('loads records, inventories PTYs, restores ownership, then projects tabs exactly once', async () => { + it('answers the tab restore from the index while reconcile and the startup pass hang', async () => { const runtime = new OrcaRuntimeService() const hydrate = vi.fn() - const refresh = vi.fn(async () => new Set()) - const ensureHost = vi.fn(async () => undefined) - const reconcileRestartLeases = vi.fn(async () => undefined) - const restoreReadableSessions = vi.fn(async () => undefined) - const internal = runtime as unknown as { - hasPersistedStructuredAgentSessionStore(): boolean - getKnownWorkspaceSessionWorktreeIds(): Set - hydrateHeadlessMobileSessionTabsFromWorkspaceSession( - worktreeId?: string, - options?: { allowAttachedWindow?: boolean; onlyRuntimeOwnedTerminals?: boolean } - ): Set - refreshMobileSessionPtyRecords(): Promise | null> - ensureStructuredAgentSessionHost(): Promise - } + const reconcileRestartLeases = vi.fn(() => new Promise(() => undefined)) + const restoreStartupSessions = vi.fn(() => new Promise(() => undefined)) + const setSessionTabVisibility = vi.fn(async () => undefined) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as RestoreInternals internal.hasPersistedStructuredAgentSessionStore = () => true internal.getKnownWorkspaceSessionWorktreeIds = () => new Set(['workspace-1']) internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = hydrate - internal.refreshMobileSessionPtyRecords = refresh - internal.ensureStructuredAgentSessionHost = ensureHost + internal.refreshMobileSessionPtyRecords = async () => new Set() + internal.ensureStructuredAgentSessionHost = async () => undefined + const store = vi.spyOn(internal, 'storeMobileSessionSnapshot') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test reads only the host members stubbed here. setStructuredAgentSessionHost({ reconcileRestartLeases, - restoreReadableSessions, - listSessionTabs: () => [] + restoreStartupSessions, + setSessionTabVisibility, + listVisibleSessionIds: () => ['a', 'b', 'c'], + listPersistedSessionTabs: (ids: readonly string[]) => + ids.map((sessionId) => ({ + sessionId, + workspaceId: sessionId === 'c' ? 'workspace-2' : 'workspace-1', + agent: 'codex' + })) } as never) const first = runtime.restoreStructuredAgentSessionTabs() const second = runtime.restoreStructuredAgentSessionTabs() expect(second).toBe(first) - await Promise.all([first, second]) + await first expect(hydrate).toHaveBeenCalledWith('workspace-1', { allowAttachedWindow: true, onlyRuntimeOwnedTerminals: true }) - expect(hydrate).toHaveBeenCalledWith() - expect(refresh).toHaveBeenCalledOnce() - expect(reconcileRestartLeases).toHaveBeenCalledOnce() - expect(restoreReadableSessions).toHaveBeenCalledOnce() - expect(ensureHost).toHaveBeenCalledOnce() - expect(ensureHost.mock.invocationCallOrder[0]).toBeLessThan( - refresh.mock.invocationCallOrder[0] ?? Infinity - ) - expect(refresh.mock.invocationCallOrder[0]).toBeLessThan( - reconcileRestartLeases.mock.invocationCallOrder[0] ?? Infinity - ) - expect(reconcileRestartLeases.mock.invocationCallOrder[0]).toBeLessThan( - restoreReadableSessions.mock.invocationCallOrder[0] ?? Infinity - ) - expect(restoreReadableSessions.mock.invocationCallOrder[0]).toBeLessThan( - hydrate.mock.invocationCallOrder[0] ?? Infinity - ) - }) - - it('prefers the durable visible-session index after a legacy profile drops agent tabs', async () => { - const runtime = new OrcaRuntimeService() - const restoreReadableSessions = vi.fn(async () => undefined) - const internal = runtime as unknown as { - store: { getWorkspaceSession: () => unknown } - hasPersistedStructuredAgentSessionStore(): boolean - getKnownWorkspaceSessionWorktreeIds(): Set - hydrateHeadlessMobileSessionTabsFromWorkspaceSession(): Set - refreshMobileSessionPtyRecords(): Promise | null> - ensureStructuredAgentSessionHost(): Promise - } - internal.store = { - getWorkspaceSession: () => ({ - activeRepoId: null, - activeWorktreeId: 'workspace-1', - activeTabId: null, - tabsByWorktree: {}, - terminalLayoutsByTabId: {}, - unifiedTabs: { 'workspace-1': [] } - }) - } - internal.hasPersistedStructuredAgentSessionStore = () => true - internal.getKnownWorkspaceSessionWorktreeIds = () => new Set() - internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() - internal.refreshMobileSessionPtyRecords = async () => new Set() - internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ - reconcileRestartLeases: async () => undefined, - getPersistedVisibleSessionTabIndex: () => ({ - present: true, - sessionIds: ['session-survives-rollback'] - }), - restoreReadableSessions, - listSessionTabs: () => [] - } as never) - - await runtime.restoreStructuredAgentSessionTabs() - - expect(restoreReadableSessions).toHaveBeenCalledWith(['session-survives-rollback']) + expect(restoreStartupSessions).toHaveBeenCalledOnce() + // One store write per workspace, and no visibility write: the ids came from the index. + expect(store.mock.calls.map(([workspaceId]) => workspaceId)).toEqual([ + 'workspace-1', + 'workspace-2' + ]) + expect(setSessionTabVisibility).not.toHaveBeenCalled() + const listed = await runtime.listMobileSessionTabs('id:workspace-1') + expect(listed.tabs.map((tab) => tab.id)).toEqual(['agent-session:a', 'agent-session:b']) }) - it('treats an empty durable visible-session index as authoritative', async () => { + it('lists exactly what the durable index holds, including nothing', async () => { const runtime = new OrcaRuntimeService() - const restoreReadableSessions = vi.fn(async () => undefined) - const internal = runtime as unknown as { - store: { getWorkspaceSession: () => unknown } - hasPersistedStructuredAgentSessionStore(): boolean - getKnownWorkspaceSessionWorktreeIds(): Set - hydrateHeadlessMobileSessionTabsFromWorkspaceSession(): Set - refreshMobileSessionPtyRecords(): Promise | null> - ensureStructuredAgentSessionHost(): Promise - } - internal.store = { - getWorkspaceSession: () => ({ - activeRepoId: null, - activeWorktreeId: 'workspace-1', - activeTabId: 'agent-session:closed-session', - tabsByWorktree: {}, - terminalLayoutsByTabId: {}, - activeTabIdByWorktree: { 'workspace-1': 'agent-session:closed-session' }, - unifiedTabs: { - 'workspace-1': [ - { - id: 'agent-session:closed-session', - entityId: 'closed-session', - groupId: 'group-1', - worktreeId: 'workspace-1', - contentType: 'agent-session', - label: 'Codex Chat', - customLabel: null, - color: null, - sortOrder: 0, - createdAt: 1 - } - ] - } - }) - } + const listPersistedSessionTabs = vi.fn(() => []) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as RestoreInternals + internal.store = { getWorkspaceSession: () => savedSessionWithChat('closed-session') } internal.hasPersistedStructuredAgentSessionStore = () => true internal.getKnownWorkspaceSessionWorktreeIds = () => new Set() internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test reads only the host members stubbed here. setStructuredAgentSessionHost({ reconcileRestartLeases: async () => undefined, - getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: [] }), - restoreReadableSessions, - listSessionTabs: () => [] + restoreStartupSessions: async () => undefined, + listVisibleSessionIds: () => [], + listPersistedSessionTabs } as never) await runtime.restoreStructuredAgentSessionTabs() - expect(restoreReadableSessions).toHaveBeenCalledWith([]) + expect(listPersistedSessionTabs).toHaveBeenCalledExactlyOnceWith([]) }) it('normalizes a restored tab id and removes it when closed', async () => { @@ -223,12 +187,14 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test reads only the host members stubbed here. setStructuredAgentSessionHost({ reconcileRestartLeases: async () => undefined, - restoreReadableSessions: async () => undefined, + restoreStartupSessions: async () => undefined, close: closeStructuredSession, setSessionTabVisibility, - listSessionTabs: () => [ + listVisibleSessionIds: () => ['restored'], + listPersistedSessionTabs: () => [ { sessionId: 'agent-session:agent-session:restored-session', workspaceId: 'workspace-1', @@ -327,23 +293,19 @@ describe('structured session cold restoration', () => { it('publishes restored Claude tabs with the Claude title', async () => { const runtime = new OrcaRuntimeService() - const publish = vi.spyOn(runtime, 'publishStructuredAgentSessionTab') - const internal = runtime as unknown as { - hasPersistedStructuredAgentSessionStore(): boolean - getKnownWorkspaceSessionWorktreeIds(): Set - hydrateHeadlessMobileSessionTabsFromWorkspaceSession(): Set - refreshMobileSessionPtyRecords(): Promise | null> - ensureStructuredAgentSessionHost(): Promise - } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as RestoreInternals internal.hasPersistedStructuredAgentSessionStore = () => true internal.getKnownWorkspaceSessionWorktreeIds = () => new Set() internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test reads only the host members stubbed here. setStructuredAgentSessionHost({ reconcileRestartLeases: async () => undefined, - restoreReadableSessions: async () => undefined, - listSessionTabs: () => [ + restoreStartupSessions: async () => undefined, + listVisibleSessionIds: () => ['restored'], + listPersistedSessionTabs: () => [ { sessionId: 'agent-session:agent-session:restored-claude', workspaceId: 'workspace-1', @@ -354,25 +316,17 @@ describe('structured session cold restoration', () => { await runtime.restoreStructuredAgentSessionTabs() - expect(publish).toHaveBeenCalledWith({ - workspaceId: 'workspace-1', - sessionId: 'restored-claude', - agent: 'claude', - activate: false, - notify: false - }) - const restored = await runtime.listMobileSessionTabs('id:workspace-1') - expect(restored.tabs).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - type: 'agent-session', - id: 'agent-session:restored-claude', - title: 'Claude Chat', - agent: 'claude' - }) - ]) - ) + expect(restored.tabs).toEqual([ + expect.objectContaining({ + type: 'agent-session', + id: 'agent-session:restored-claude', + sessionId: 'restored-claude', + title: 'Claude Chat', + agent: 'claude', + isActive: false + }) + ]) }) it('commits the host close when the renderer already removed the structured tab', async () => { diff --git a/src/main/runtime/orca-runtime-tests/terminal-handles-and-agent-status-part-02.spec.ts b/src/main/runtime/orca-runtime-tests/terminal-handles-and-agent-status-part-02.spec.ts index 237afc57bc96..be454a5bde84 100644 --- a/src/main/runtime/orca-runtime-tests/terminal-handles-and-agent-status-part-02.spec.ts +++ b/src/main/runtime/orca-runtime-tests/terminal-handles-and-agent-status-part-02.spec.ts @@ -142,7 +142,7 @@ describe('OrcaRuntimeService', () => { syncSinglePty(runtime, 'pty-1', { paneTitle: 'claude agents' }) const [terminal] = (await runtime.listTerminals()).terminals - expect(runtime.getAgentStatusForHandle(terminal.handle)).toBeNull() + await expect(runtime.getAgentStatusForHandle(terminal.handle)).resolves.toBeNull() }) it('lists live terminals with fresh pane titles over stale tab titles', async () => { diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts index 439aca59b017..064ae577e0f1 100644 --- a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-transfer.ts @@ -81,14 +81,30 @@ export function workerTerminalResourceHasIdentityConflict( ) } +/** Every resource whose process incarnation starts with `prefix`, newest first. */ +export function listWorkerTerminalResourcesByIncarnationPrefix( + this: OrchestrationDb, + prefix: string +): WorkerTerminalResourceRow[] { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table is exactly its row shape. + return this.db + .prepare( + `SELECT * FROM worker_terminal_resources + WHERE substr(process_incarnation, 1, ?) = ? ORDER BY updated_at DESC` + ) + .all(prefix.length, prefix) as WorkerTerminalResourceRow[] +} + export type WorkerTerminalTransferMethods = { findTransferableWorkerTerminalResource: typeof findTransferableWorkerTerminalResource workerTerminalResourceHasIdentityConflict: typeof workerTerminalResourceHasIdentityConflict + listWorkerTerminalResourcesByIncarnationPrefix: typeof listWorkerTerminalResourcesByIncarnationPrefix } export function attachWorkerTerminalTransfer(ctor: { prototype: object }): void { Object.assign(ctor.prototype, { findTransferableWorkerTerminalResource, - workerTerminalResourceHasIdentityConflict + workerTerminalResourceHasIdentityConflict, + listWorkerTerminalResourcesByIncarnationPrefix }) } diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts index 048fc7732e40..52103cd5322b 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts @@ -109,7 +109,7 @@ function harness(options: { resolveStructuredTarget: (mailboxHandle) => mailboxHandle === mailbox ? { sessionId: IDENTITY.sessionId, dispatchId } : null, host: { - readGateFacts: () => (journal === null ? null : structuredSessionGateFacts(journal)), + readGateFacts: async () => (journal === null ? null : structuredSessionGateFacts(journal)), currentFence: () => 4, send } @@ -329,7 +329,7 @@ describe('forgetting one settled worker', () => { : null }, host: { - readGateFacts: () => structuredSessionGateFacts(journal), + readGateFacts: async () => structuredSessionGateFacts(journal), currentFence: () => 4, send } diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts index 62a555d41191..d3afed4923f0 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts @@ -49,8 +49,8 @@ export type StructuredPointerSendOutcome = | { kind: 'unattached' } export type StructuredMailboxPointerHost = { - /** The idle gate, read off the session's full reduced timeline; `null` when it is not attached. */ - readGateFacts: (sessionId: string) => StructuredSessionGateFacts | null + /** The idle gate, read off the session's full reduced timeline; `null` when it cannot be read. */ + readGateFacts: (sessionId: string) => Promise send: (input: { sessionId: string dispatchId: string | null @@ -184,7 +184,7 @@ export class OrchestrationStructuredMailboxPointerDelivery< reservedTypes: ReadonlySet | undefined ): Promise { const sessionId = target.sessionId - const session = this.deps.host.readGateFacts(sessionId) + const session = await this.deps.host.readGateFacts(sessionId) const decision = decideStructuredSessionPointerDelivery({ session }) if (!decision.deliver) { this.retain(mailboxHandle, sessionId, decision.retain, reservedTypes) diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts index 91bf1158e077..ba28ee431743 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts @@ -38,7 +38,7 @@ describe('structured mailbox pointer host', () => { hostRef.current = null }) - it('reads the gate facts from the FULL timeline, never a bounded tail', () => { + it('reads the gate facts from the FULL timeline, never a bounded tail', async () => { // The defect this pins: a running turn is announced by ONE lifecycle item, and settlement // tombstones it rather than rewriting it. A long tool-calling turn pushes that item arbitrarily // far from the tail, so any page-sized read reports a busy worker as idle — and the pointer is @@ -46,22 +46,22 @@ describe('structured mailbox pointer host', () => { // it -- either way folded into work already in flight rather than read as a new instruction. const items = [runningTurn(), ...transcript(500)] hostRef.current = { journalSnapshot: () => ({ items }) } - expect(createStructuredMailboxPointerHost().readGateFacts('s1')).toEqual({ + expect(await createStructuredMailboxPointerHost().readGateFacts('s1')).toEqual({ turnRunning: true, awaitingHuman: false }) }) - it('answers null rather than idle when the session cannot be read', () => { + it('answers null rather than idle when the session cannot be read', async () => { // Null retains the pointer; `{turnRunning:false}` would deliver a nudge into a session this // runtime cannot see at all. - expect(createStructuredMailboxPointerHost().readGateFacts('s1')).toBeNull() + expect(await createStructuredMailboxPointerHost().readGateFacts('s1')).toBeNull() hostRef.current = { journalSnapshot: () => { throw new Error('agent_session_ownership_unknown') } } - expect(createStructuredMailboxPointerHost().readGateFacts('s1')).toBeNull() + expect(await createStructuredMailboxPointerHost().readGateFacts('s1')).toBeNull() }) it('reports an unattached host rather than a rejection when nothing can be sent', async () => { @@ -91,7 +91,7 @@ describe('structured mailbox pointer host', () => { value: { submission: { dispatchState } } }) ) - hostRef.current = { send } + hostRef.current = { send, waitForSendSettlement: async () => undefined } await expect( createStructuredMailboxPointerHost().send({ sessionId: 's1', @@ -107,6 +107,28 @@ describe('structured mailbox pointer host', () => { expect(send.mock.calls[0]![1]!.retryUnknown).toBeUndefined() }) + it('consumes mail once an accepted nudge is delivered while the worker starts (W10)', async () => { + hostRef.current = { + send: async () => ({ + ok: true, + value: { clientMessageId: 'op1', submission: { dispatchState: 'pending' } } + }), + waitForSendSettlement: async () => ({ + value: { clientMessageId: 'op1', submission: { dispatchState: 'accepted' } } + }) + } + await expect( + createStructuredMailboxPointerHost().send({ + sessionId: 's1', + dispatchId: 'd1', + operationId: 'op1', + expectedRuntimeFence: 1, + payloadFingerprint: 'fp', + body: { kind: 'message', role: 'user', blocks: [] } + } as never) + ).resolves.toEqual({ kind: 'sent', state: 'accepted' }) + }) + it('scopes direct peer mail to the session when there is no dispatch to scope to', async () => { // Direct mail is addressed to the worker's own handle, so there may be no dispatch at all. // The ledger is keyed on (callerKey, operationId): a key derived from the session keeps that diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts index b722952df92f..d53dc04600e7 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts @@ -6,6 +6,7 @@ * the send and reports what the host said. */ +import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../shared/orchestration-timing-budgets' import { AGENT_SESSION_NOT_ATTACHED } from '../../native-chat/agent-session-wire/structured-agent-session-mutation-admission' import { getStructuredAgentSessionHost } from '../../native-chat/agent-session-wire/structured-agent-session-registry' import type { StructuredMailboxPointerHost } from './structured-mailbox-pointer-delivery' @@ -38,15 +39,16 @@ export function structuredSessionPointerCallerKey(sessionId: string): string { * idle-with-history is the normal steady state of a working agent. Shared so the pointer lane and * group addressing cannot disagree about it. */ -export function readStructuredSessionGateFacts( +export async function readStructuredSessionGateFacts( sessionId: string -): StructuredSessionGateFacts | null { +): Promise { const host = getStructuredAgentSessionHost() if (!host) { return null } try { - return structuredSessionGateFacts(host.journalSnapshot(sessionId).items) + // Opens a conversation the idle sweep closed; that starts no agent. + return structuredSessionGateFacts((await host.journalSnapshot(sessionId)).items) } catch (error) { // Not attached is a retain reason, not a failure; anything else is still unreadable. if ((error as Error)?.message !== AGENT_SESSION_NOT_ATTACHED.code) { @@ -94,8 +96,19 @@ export function createStructuredMailboxPointerHost(): StructuredMailboxPointerHo ? { kind: 'unattached' } : { kind: 'sent', state: 'rejected' } } - // `pending` is not yet an acknowledgement; only `accepted` may consume mail. - const state = result.value.submission.dispatchState + // `pending` is not yet an acknowledgement; only `accepted` may consume mail. Accepted is not + // delivered, so wait out a start; a wait that runs out parks for the next journal edge. + const submission = + result.value.submission.dispatchState === 'pending' + ? (( + await host + .waitForSendSettlement(input.sessionId, result.value.clientMessageId, { + budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS + }) + .catch(() => undefined) + )?.value.submission ?? result.value.submission) + : result.value.submission + const state = submission.dispatchState return { kind: 'sent', state: state === 'accepted' ? 'accepted' : state === 'rejected' ? 'rejected' : 'unknown' diff --git a/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts b/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts index 31ee011193ae..29013cecc7c2 100644 --- a/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts +++ b/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts @@ -50,6 +50,7 @@ function installHost(options: { items?: AgentJournalRenderItem[] lease?: { runtimeKind: string; claimStatus: string } hasSession?: boolean + tabListed?: boolean }): void { const lease = options.lease ?? { runtimeKind: 'native', claimStatus: 'live' } hostRef.current = { @@ -65,21 +66,42 @@ function installHost(options: { } }, hasSession: () => options.hasSession ?? true, - journalSnapshot: () => ({ items: options.items ?? [idleTurn()] }) + listVisibleSessionIds: () => (options.tabListed ? [SESSION_ID] : []), + journalSnapshot: async () => ({ items: options.items ?? [idleTurn()] }) } } +/** The durable worker-terminal rows group addressing enumerates. */ +const rows: { + terminal_handle: string + pane_key: string + process_incarnation: string + worktree_id: string + host_scope: string +}[] = [] + +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: group addressing reads only this one query off the database. +const workerDb = { listWorkerTerminalResourcesByIncarnationPrefix: () => rows } as never + function registerWorker(worktreeId = 'wt_1'): string { const handle = mintStructuredWorkerHandle() + const paneKey = mintStructuredWorkerPaneKey(SESSION_ID) structuredWorkerIdentities.register({ handle, sessionId: SESSION_ID, agent: 'codex', - paneKey: mintStructuredWorkerPaneKey(SESSION_ID), + paneKey, processIncarnation: structuredWorkerProcessIncarnation(SESSION_ID), worktreeId, hostScope: { kind: 'local', hostId: 'local' } }) + rows.push({ + terminal_handle: handle, + pane_key: paneKey, + process_incarnation: structuredWorkerProcessIncarnation(SESSION_ID), + worktree_id: worktreeId, + host_scope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) return handle } @@ -88,22 +110,28 @@ const PTY_TERMINAL = { handle: 'term_a', worktreeId: 'wt_1', agentIdentity: 'cla describe('group addressing and structured workers', () => { beforeEach(() => { structuredWorkerIdentities.clear() + rows.length = 0 hostRef.current = null }) it('enumerates a live structured worker as a candidate', () => { const handle = registerWorker() installHost({}) - expect(listAddressableStructuredWorkers()).toEqual([ + expect(listAddressableStructuredWorkers(workerDb)).toEqual([ { handle, worktreeId: 'wt_1', agentIdentity: 'codex' } ]) }) - it('leaves out a worker whose session is not proven live', () => { - // Addressing a settled worker would store mail no lane will ever deliver. - registerWorker() - installHost({ lease: { runtimeKind: 'native', claimStatus: 'live' }, hasSession: false }) - expect(listAddressableStructuredWorkers()).toEqual([]) + it('keeps a worker at rest and leaves out a retired one', () => { + // At rest: its agent was stopped while idle, its chat tab still lists it, and mail starts it. + const handle = registerWorker() + installHost({ lease: { runtimeKind: 'native', claimStatus: 'released' }, tabListed: true }) + expect(listAddressableStructuredWorkers(workerDb)).toEqual([ + { handle, worktreeId: 'wt_1', agentIdentity: 'codex' } + ]) + // Retired: released with its tab gone. Addressing it would store mail no lane delivers. + installHost({ lease: { runtimeKind: 'native', claimStatus: 'released' }, tabListed: false }) + expect(listAddressableStructuredWorkers(workerDb)).toEqual([]) }) it('reaches a structured worker through @all', () => { @@ -112,14 +140,14 @@ describe('group addressing and structured workers', () => { // machinery never ran and the sender got exit 0 with a receipt naming only who did resolve. const handle = registerWorker() installHost({}) - const recipients = [PTY_TERMINAL, ...listAddressableStructuredWorkers()] + const recipients = [PTY_TERMINAL, ...listAddressableStructuredWorkers(workerDb)] expect(resolveGroupAddress('@all', 'term_sender', recipients, () => 'idle')).toContain(handle) }) it('reaches a structured worker through @worktree: and @codex, but not @claude', () => { const handle = registerWorker('wt_2') installHost({}) - const recipients = [PTY_TERMINAL, ...listAddressableStructuredWorkers()] + const recipients = [PTY_TERMINAL, ...listAddressableStructuredWorkers(workerDb)] expect(resolveGroupAddress('@worktree:wt_2', 'term_sender', recipients, () => 'idle')).toEqual([ handle ]) @@ -129,20 +157,20 @@ describe('group addressing and structured workers', () => { ]) }) - it('reads @idle status off the FULL timeline, never a bounded tail', () => { + it('reads @idle status off the FULL timeline, never a bounded tail', async () => { // The same trap that already cost this branch once: settlement tombstones the lifecycle item // rather than rewriting it, so a long tool-calling turn pushes it arbitrarily far from the // tail and any page-sized read reports a BUSY worker as idle — then `@idle` broadcasts into a // running turn, which Codex refuses outright and Claude queues behind. registerWorker() installHost({ items: [runningTurn(), ...transcript(500)] }) - expect(structuredWorkerAgentStatus(SESSION_ID)).toBe('working') + expect(await structuredWorkerAgentStatus(SESSION_ID)).toBe('working') }) - it('answers idle only when no turn is running and no human is awaited', () => { + it('answers idle only when no turn is running and no human is awaited', async () => { registerWorker() installHost({ items: [idleTurn()] }) - expect(structuredWorkerAgentStatus(SESSION_ID)).toBe('idle') + expect(await structuredWorkerAgentStatus(SESSION_ID)).toBe('idle') installHost({ items: [ { @@ -156,26 +184,27 @@ describe('group addressing and structured workers', () => { } as unknown as AgentJournalRenderItem ] }) - expect(structuredWorkerAgentStatus(SESSION_ID)).toBe('attention') + expect(await structuredWorkerAgentStatus(SESSION_ID)).toBe('attention') }) - it('answers null rather than idle when the session cannot be read', () => { + it('answers null rather than idle when the session cannot be read', async () => { // Unknown must never read as idle, or `@idle` wakes a worker mid-turn. hostRef.current = null - expect(structuredWorkerAgentStatus(SESSION_ID)).toBeNull() + expect(await structuredWorkerAgentStatus(SESSION_ID)).toBeNull() }) }) describe('sendGroupMessage actually composes structured workers in', () => { beforeEach(() => { structuredWorkerIdentities.clear() + rows.length = 0 hostRef.current = null }) /** * Drives the real `sendGroupMessage`, not `resolveGroupAddress`. * - * The suite above hand-composed `[PTY_TERMINAL, ...listAddressableStructuredWorkers()]` itself, + * The suite above hand-composed `[PTY_TERMINAL, ...listAddressableStructuredWorkers(workerDb)]` itself, * so deleting the composition at the call site left it green — the exact regression the fix * describes could come straight back. This test owns that seam. */ @@ -189,6 +218,7 @@ describe('sendGroupMessage actually composes structured workers in', () => { getCurrentRunForCoordinator: () => undefined, getActiveDispatchMailboxOwners: () => [], getRunMailboxOwnerIdsForHandle: () => [], + listWorkerTerminalResourcesByIncarnationPrefix: () => rows, insertMessages: (rows: { to: string }[]) => { inserted.push(...rows) return rows.map((row, index) => ({ id: `m${index}`, to_handle: row.to, type: 'status' })) @@ -198,7 +228,7 @@ describe('sendGroupMessage actually composes structured workers in', () => { // No PTY terminals at all: if the call site does not compose structured workers in, the // group resolves empty and this throws instead of delivering. listTerminals: async () => ({ terminals: [] }), - getAgentStatusForHandle: () => 'idle', + getAgentStatusForHandle: async () => 'idle', getLiveTerminalPaneKey: () => structuredWorkerIdentities.get(handle)!.paneKey, notifyMessageArrived: () => {} } @@ -232,6 +262,7 @@ describe('sendGroupMessage actually composes structured workers in', () => { getActiveDispatchForIdentity: () => ({ run_id: 'run_1' }), getActiveDispatchMailboxOwners: () => [], getRunMailboxOwnerIdsForHandle: () => [], + listWorkerTerminalResourcesByIncarnationPrefix: () => rows, listWorkerTerminalResources: () => [ { dispatchId: 'ctx_structured', @@ -259,7 +290,7 @@ describe('sendGroupMessage actually composes structured workers in', () => { listTerminals: async () => ({ terminals: [{ handle: 'term_claude', worktreeId: 'wt_1', agentIdentity: 'claude' }] }), - getAgentStatusForHandle: () => 'idle', + getAgentStatusForHandle: async () => 'idle', getLiveTerminalPaneKey: () => null, getOrchestrationDb: () => db, notifyMessageArrived: () => {} diff --git a/src/main/runtime/orchestration/structured-worker-group-addressing.ts b/src/main/runtime/orchestration/structured-worker-group-addressing.ts index 168bd8701188..4115f168552b 100644 --- a/src/main/runtime/orchestration/structured-worker-group-addressing.ts +++ b/src/main/runtime/orchestration/structured-worker-group-addressing.ts @@ -16,8 +16,13 @@ */ import type { TuiAgent } from '../../../shared/tui-agent' -import { observeStructuredWorker, structuredWorkerAgent } from '../structured-worker-authority' -import { structuredWorkerIdentities } from '../structured-worker-identity' +import { structuredWorkerAgent } from '../structured-worker-authority' +import { structuredWorkerAddressable } from '../structured-worker-custody' +import { + STRUCTURED_WORKER_INCARNATION_PREFIX, + structuredWorkerIdentityFromRow +} from '../structured-worker-identity' +import type { OrchestrationDb } from './db' import { readStructuredSessionGateFacts } from './structured-mailbox-pointer-host' /** The only facts group addressing reads off a recipient. */ @@ -29,15 +34,29 @@ export type OrchestrationAddressableAgent = { } /** - * Live structured workers of this runtime, as group-address candidates. + * Structured workers this runtime owns, as group-address candidates. * - * Liveness-gated on the same observation the rest of the structured surface uses: a settled or - * handed-off worker is not a recipient, and addressing one would store mail no lane will deliver. + * Read from the durable worker-terminal rows, which outlive a settled dispatch and a restart, and + * gated on ownership and on the orchestration not having released it — the same answer direct mail + * routes on — never on liveness: a worker at rest is a recipient, and the mail starts it. A retired + * or released one is not. */ -export function listAddressableStructuredWorkers(): OrchestrationAddressableAgent[] { - return structuredWorkerIdentities - .list() - .filter((identity) => observeStructuredWorker(identity).status === 'live') +export function listAddressableStructuredWorkers( + db: OrchestrationDb | null +): OrchestrationAddressableAgent[] { + const seen = new Set() + return ( + db?.listWorkerTerminalResourcesByIncarnationPrefix(STRUCTURED_WORKER_INCARNATION_PREFIX) ?? [] + ) + .flatMap((row) => { + const identity = structuredWorkerIdentityFromRow(row) + // Newest row first: a session is one recipient, under its latest handle. + if (!identity || seen.has(identity.sessionId)) { + return [] + } + seen.add(identity.sessionId) + return structuredWorkerAddressable(db, identity.sessionId, row) === true ? [identity] : [] + }) .map((identity) => ({ handle: identity.handle, worktreeId: identity.worktreeId, @@ -52,8 +71,8 @@ export function listAddressableStructuredWorkers(): OrchestrationAddressableAgen * would wake a worker mid-turn — which Codex coalesces into the running turn and Claude queues * behind it. */ -export function structuredWorkerAgentStatus(sessionId: string): string | null { - const facts = readStructuredSessionGateFacts(sessionId) +export async function structuredWorkerAgentStatus(sessionId: string): Promise { + const facts = await readStructuredSessionGateFacts(sessionId) if (!facts) { return null } diff --git a/src/main/runtime/orchestration/structured-worker-journal-page.ts b/src/main/runtime/orchestration/structured-worker-journal-page.ts index 35676aaf52cc..8d260339fa6f 100644 --- a/src/main/runtime/orchestration/structured-worker-journal-page.ts +++ b/src/main/runtime/orchestration/structured-worker-journal-page.ts @@ -16,14 +16,17 @@ export type StructuredJournalPage = { hasOlder: boolean } -/** The newest page of a session's journal, or null when this runtime cannot read it. */ -export function readStructuredJournalPage(sessionId: string): StructuredJournalPage | null { +/** The newest page of a session's journal, or null when this runtime cannot read it. A closed + * conversation is opened for the read; that starts no agent. */ +export async function readStructuredJournalPage( + sessionId: string +): Promise { const host = getStructuredAgentSessionHost() if (!host) { return null } try { - const result = host.history({ + const result = await host.history({ sessionId, direction: 'tail', limit: STRUCTURED_JOURNAL_PAGE_LIMIT diff --git a/src/main/runtime/orchestration/worker-output-archive.ts b/src/main/runtime/orchestration/worker-output-archive.ts index d7e894cf54c1..ddae515a7f2c 100644 --- a/src/main/runtime/orchestration/worker-output-archive.ts +++ b/src/main/runtime/orchestration/worker-output-archive.ts @@ -91,7 +91,7 @@ export async function captureWorkerOutputArchive(args: { structuredWorker?: StructuredWorkerIdentity | null }): Promise { if (args.structuredWorker) { - const content = captureStructuredWorkerArchive( + const content = await captureStructuredWorkerArchive( args.structuredWorker, structuredWorkerAgent(args.structuredWorker) ) diff --git a/src/main/runtime/rpc/methods/agent-launch-structured-prompt.ts b/src/main/runtime/rpc/methods/agent-launch-structured-prompt.ts index 0037d6fc35e9..11762137ba2f 100644 --- a/src/main/runtime/rpc/methods/agent-launch-structured-prompt.ts +++ b/src/main/runtime/rpc/methods/agent-launch-structured-prompt.ts @@ -63,9 +63,9 @@ export async function commitStructuredAgentSessionLaunchPrompt(args: { // Settlement can fail after the journal append. Re-read the authoritative row before asking // the caller to resend, otherwise a retry creates a duplicate turn. try { - const committed = args.host - .journalSnapshot(args.sessionId) - .submissions.find((submission) => submission.clientMessageId === clientMessageId) + const committed = (await args.host.journalSnapshot(args.sessionId)).submissions.find( + (submission) => submission.clientMessageId === clientMessageId + ) if (committed) { return clientMessageId } diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.test.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.test.ts index a0d0426509b7..affc8646e32d 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.test.ts @@ -47,6 +47,7 @@ function installHost(options: { close?: () => Promise setSessionTabVisibility?: () => Promise historyThrows?: boolean + tabListed?: boolean }) { const record = options.record === undefined @@ -63,6 +64,7 @@ function installHost(options: { let closed = false hostRef.current = { deps: { store: { getRecord: () => record } }, + listVisibleSessionIds: () => (options.tabListed ? [IDENTITY.sessionId] : []), hasSession: () => (closed ? false : (options.hasSession ?? true)), setSessionTabVisibility: options.setSessionTabVisibility ?? (async () => {}), close: @@ -84,7 +86,7 @@ describe('structured worker observation', () => { hostRef.current = null }) - it('is unverifiable, never exited, when the host is not installed', () => { + it('is unverifiable, never exited, when the host is not installed', async () => { // Not being able to look is not a death certificate. expect(observeStructuredWorker(IDENTITY)).toEqual({ status: 'unverifiable', @@ -92,12 +94,12 @@ describe('structured worker observation', () => { }) }) - it('is live when the host holds the session under a live native lease', () => { + it('is live when the host holds the session under a live native lease', async () => { installHost({}) expect(observeStructuredWorker(IDENTITY).status).toBe('live') }) - it('is exited only on a released lease with death evidence', () => { + it('is exited only on a released lease with death evidence', async () => { installHost({ claimStatus: 'released', deathEvidence: { kind: 'exit-observed', detail: 'x', observedAt: 1 } @@ -105,7 +107,7 @@ describe('structured worker observation', () => { expect(observeStructuredWorker(IDENTITY).status).toBe('exited') }) - it('is unverifiable when a terminal an older build recorded holds the lease', () => { + it('is unverifiable when a terminal an older build recorded holds the lease', async () => { installHost({ claimStatus: 'conflicted' }) expect(observeStructuredWorker(IDENTITY).status).toBe('unverifiable') }) @@ -127,22 +129,30 @@ describe('structured worker stop', () => { }) }) - it.each([ - { hasSession: false }, - { claimStatus: 'conflicted' }, - { claimStatus: 'released' }, - { record: null } - ])('retains without positive exit evidence: %j', async (options) => { - installHost({ ...options, close: async () => {} }) - const retireStructuredAgentSessionTabFromSnapshot = vi.fn() - const result = await stopStructuredWorker(IDENTITY, 'd1', { - forgetStructuredSessionMail: vi.fn(), - retireStructuredAgentSessionTabFromSnapshot + it('settles a released lease whose stop was unproven: nothing is left running to close', async () => { + // A release that could not prove its stop sent no signal and left the verdict `unverifiable`; + // closing the chat is the user's action, so that bookkeeping does not refuse it. + installHost({ claimStatus: 'released', close: async () => {} }) + await expect(stopStructuredWorker(IDENTITY, 'd1')).resolves.toEqual({ + stopped: true, + closeAttempted: true }) - expect(result).toMatchObject({ stopped: false, closeAttempted: true }) - expect(retireStructuredAgentSessionTabFromSnapshot).not.toHaveBeenCalled() }) + it.each([{ hasSession: false }, { claimStatus: 'conflicted' }, { record: null }])( + 'retains without positive exit evidence: %j', + async (options) => { + installHost({ ...options, close: async () => {} }) + const retireStructuredAgentSessionTabFromSnapshot = vi.fn() + const result = await stopStructuredWorker(IDENTITY, 'd1', { + forgetStructuredSessionMail: vi.fn(), + retireStructuredAgentSessionTabFromSnapshot + }) + expect(result).toMatchObject({ stopped: false, closeAttempted: true }) + expect(retireStructuredAgentSessionTabFromSnapshot).not.toHaveBeenCalled() + } + ) + it('retains when the close throws, and admits the close was issued', async () => { installHost({ close: async () => { @@ -191,9 +201,9 @@ describe('structured worker output', () => { hostRef.current = null }) - it('round-trips the journal through the archive and back out of a released read', () => { + it('round-trips the journal through the archive and back out of a released read', async () => { installHost({}) - const live = readStructuredWorkerJournal({ + const live = await readStructuredWorkerJournal({ identity: IDENTITY, dispatchId: 'd1', workerState: 'ready', @@ -201,7 +211,7 @@ describe('structured worker output', () => { agent: 'claude' }) expect(live.source).toBe('transcript') - const archive = captureStructuredWorkerArchive(IDENTITY, 'claude') + const archive = await captureStructuredWorkerArchive(IDENTITY, 'claude') hostRef.current = null const archived = readArchivedStructuredJournal({ dispatchId: 'd1', @@ -219,7 +229,7 @@ describe('structured worker output', () => { expect(archived.sourceIdentity).not.toBe(live.sourceIdentity) }) - it('redacts dispatch capabilities from the archived journal', () => { + it('redacts dispatch capabilities from the archived journal', async () => { installHost({ items: [ { @@ -233,13 +243,13 @@ describe('structured worker output', () => { } as unknown as AgentJournalRenderItem ] }) - const archive = captureStructuredWorkerArchive(IDENTITY, 'claude') + const archive = await captureStructuredWorkerArchive(IDENTITY, 'claude') expect(JSON.stringify(archive)).not.toContain('dcap_aaa') expect(JSON.stringify(archive)).toContain('[dispatch capability redacted]') }) - it('refuses to read a session the host no longer holds', () => { - expect(() => + it('refuses to read a session the host no longer holds', async () => { + await expect( readStructuredWorkerJournal({ identity: IDENTITY, dispatchId: 'd1', @@ -247,15 +257,15 @@ describe('structured worker output', () => { liveness: 'live', agent: 'claude' }) - ).toThrow(/not attached/) + ).rejects.toThrow(/not attached/) }) - it('reports an unverifiable worker as unknown, never as running', () => { + it('reports an unverifiable worker as unknown, never as running', async () => { // The `could not look, therefore it is alive` inversion. After a restart the runtime observes // `unverifiable` — no attached provider child in this generation — while the journal is still // readable, and a coordinator reading `running` waits on a worker that may already be gone. installHost({}) - const read = readStructuredWorkerJournal({ + const read = await readStructuredWorkerJournal({ identity: IDENTITY, dispatchId: 'd1', workerState: 'ready', @@ -266,9 +276,9 @@ describe('structured worker output', () => { expect(read.status.liveness).toBe('unverifiable') }) - it('carries each proven verdict through unchanged', () => { + it('carries each proven verdict through unchanged', async () => { installHost({}) - const live = readStructuredWorkerJournal({ + const live = await readStructuredWorkerJournal({ identity: IDENTITY, dispatchId: 'd1', workerState: 'ready', @@ -276,7 +286,7 @@ describe('structured worker output', () => { agent: 'claude' }) expect(live.status).toMatchObject({ terminal: 'running', liveness: 'live' }) - const exited = readStructuredWorkerJournal({ + const exited = await readStructuredWorkerJournal({ identity: IDENTITY, dispatchId: 'd1', workerState: 'succeeded', @@ -286,9 +296,9 @@ describe('structured worker output', () => { expect(exited.status).toMatchObject({ terminal: 'exited', liveness: 'exited' }) }) - it('states that a settled release is exited', () => { + it('states that a settled release is exited', async () => { installHost({}) - const archive = captureStructuredWorkerArchive(IDENTITY, 'claude') + const archive = await captureStructuredWorkerArchive(IDENTITY, 'claude') const archived = readArchivedStructuredJournal({ dispatchId: 'd1', workerState: 'succeeded', @@ -300,12 +310,12 @@ describe('structured worker output', () => { expect(archived.status).toMatchObject({ terminal: 'exited', liveness: 'exited' }) }) - it('never calls an unproven release exited', () => { + it('never calls an unproven release exited', async () => { // The archive is frozen BEFORE the close. `release_unknown` is the state that records a close // that did NOT land, and a coordinator reading `exited` there starts a replacement worker over // the same worktree while the original provider child may still be attached. installHost({}) - const archive = captureStructuredWorkerArchive(IDENTITY, 'claude') + const archive = await captureStructuredWorkerArchive(IDENTITY, 'claude') for (const releaseState of ['unknown', 'releasing'] as const) { const archived = readArchivedStructuredJournal({ dispatchId: 'd1', @@ -324,7 +334,7 @@ describe('structured worker output', () => { // `readArchivedWorkerOutput`, and the resource row it already holds is the only thing that // knows whether the close landed. installHost({}) - const archive = captureStructuredWorkerArchive(IDENTITY, 'claude') + const archive = await captureStructuredWorkerArchive(IDENTITY, 'claude') const db = { getWorkerTerminalArchive: () => ({ dispatch_id: 'd1', @@ -355,12 +365,12 @@ describe('structured worker output', () => { }) }) - it('refuses a cursor once the tail window has slid past it', () => { + it('refuses a cursor once the tail window has slid past it', async () => { // The cursor is an index into the bounded tail, and `sourceIdentity` was constant for the // worker's life, so a coordinator paging a growing journal resumed at the newest items and // skipped the middle without a word. installHost({ items: ITEMS }) - const first = readStructuredWorkerJournal({ + const first = await readStructuredWorkerJournal({ identity: IDENTITY, dispatchId: 'd1', workerState: 'ready', @@ -376,7 +386,7 @@ describe('structured worker output', () => { } as unknown as AgentJournalRenderItem ] }) - expect(() => + await expect( readStructuredWorkerJournal({ identity: IDENTITY, dispatchId: 'd1', @@ -385,7 +395,7 @@ describe('structured worker output', () => { agent: 'claude', cursor: first.cursor }) - ).toThrow(/source changed/i) + ).rejects.toThrow(/source changed/i) }) }) @@ -394,7 +404,7 @@ describe('archiving a structured worker whose journal cannot be read', () => { hostRef.current = null }) - it('settles with an empty, warned archive once the session is PROVEN gone', () => { + it('settles with an empty, warned archive once the session is PROVEN gone', async () => { // Closing the worker's chat tab is a routine user action: it evicts the child and detaches the // journal permanently. Throwing archive_failed there wedged release on evidence that could // never arrive, leaving worker-abandon as the only way out. @@ -403,7 +413,7 @@ describe('archiving a structured worker whose journal cannot be read', () => { claimStatus: 'released', deathEvidence: { kind: 'exit-observed', detail: 'surface released', observedAt: 1 } }) - const archive = captureStructuredWorkerArchive(IDENTITY, 'claude') + const archive = await captureStructuredWorkerArchive(IDENTITY, 'claude') expect(archive.messages).toEqual([]) expect(archive.processIncarnation).toBe(IDENTITY.processIncarnation) expect(archive.warnings).toContain( @@ -411,12 +421,24 @@ describe('archiving a structured worker whose journal cannot be read', () => { ) }) - it('still retains when the journal is unreadable but nothing proves the child is gone', () => { + it('retains a worker at rest: released, but its chat tab still lists it', async () => { + // Released is not retired now — the idle sweep releases a quiet worker's lease — so an owned + // worker whose journal cannot be read keeps its evidence for a later archive. + installHost({ + historyThrows: true, + claimStatus: 'released', + deathEvidence: { kind: 'exit-observed', detail: 'idle stop', observedAt: 1 }, + tabListed: true + }) + await expect(captureStructuredWorkerArchive(IDENTITY, 'claude')).rejects.toThrow(/retained/) + }) + + it('still retains when the journal is unreadable but nothing proves the child is gone', async () => { installHost({ historyThrows: true }) - expect(() => captureStructuredWorkerArchive(IDENTITY, 'claude')).toThrow(/retained/) + await expect(captureStructuredWorkerArchive(IDENTITY, 'claude')).rejects.toThrow(/retained/) }) - it('still retains when there is no host to look with', () => { - expect(() => captureStructuredWorkerArchive(IDENTITY, 'claude')).toThrow(/retained/) + it('still retains when there is no host to look with', async () => { + await expect(captureStructuredWorkerArchive(IDENTITY, 'claude')).rejects.toThrow(/retained/) }) }) diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.ts index 13d3ce1dbce1..bec15537c49d 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-lifecycle.ts @@ -40,12 +40,13 @@ import { structuredWorkerTerminalState, type StructuredWorkerObservation } from '../../structured-worker-authority' +import { structuredWorkerOwned } from '../../structured-worker-custody' import type { StructuredWorkerIdentity } from '../../structured-worker-identity' import type { WorkerTerminalReleaseState } from '../../orchestration/worker-terminal-ownership' import { releaseStructuredWorkerSession } from './orchestration-structured-worker-session' import { closeStructuredAgentSessionChild } from '../../structured-agent-session-close' -export { observeStructuredWorker, type StructuredWorkerObservation } +export { observeStructuredWorker, structuredWorkerOwned, type StructuredWorkerObservation } /** The structured worker behind a dispatch, or null when a PTY worker owns it. */ export function resolveStructuredWorkerForDispatch( @@ -83,13 +84,13 @@ export async function stopStructuredWorker( return closeStructuredAgentSessionChild(identity.sessionId, { ...(runtime ? { runtime } : {}), // Between the close and the proof, never after: an unsettled close returns early, and a - // surviving hold keeps the provider child un-evictable for the life of the app. + // surviving redrive subscription keeps nudging a session no dispatch owns. afterClose: () => releaseStructuredWorkerSession(dispatchId, runtime) }) } /** The structured half of `worker-read`, or null when a PTY worker owns the dispatch. */ -export function readStructuredWorkerOutput(args: { +export async function readStructuredWorkerOutput(args: { db: OrchestrationDb dispatchId: string workerState: string @@ -98,7 +99,7 @@ export function readStructuredWorkerOutput(args: { source?: 'auto' | 'transcript' | 'terminal' cursor?: string | number limit?: number -}): OrchestrationWorkerReadTranscriptResult | null { +}): Promise { const identity = resolveStructuredWorkerForDispatch(args.db, args.dispatchId) if (!identity) { return null @@ -123,7 +124,7 @@ export function readStructuredWorkerOutput(args: { } /** Journal page in the shape `worker-read --source transcript` already serves. */ -export function readStructuredWorkerJournal(args: { +export async function readStructuredWorkerJournal(args: { identity: StructuredWorkerIdentity dispatchId: string workerState: string @@ -131,8 +132,8 @@ export function readStructuredWorkerJournal(args: { agent: AgentType cursor?: string | number limit?: number -}): OrchestrationWorkerReadTranscriptResult { - const page = readStructuredJournalPage(args.identity.sessionId) +}): Promise { + const page = await readStructuredJournalPage(args.identity.sessionId) if (!page) { throw new OrchestrationError( 'transcript_required', @@ -213,11 +214,13 @@ function structuredJournalPrefixIdentity(args: { } /** Freezes the journal before the session is closed, so a released worker is still readable. */ -export function captureStructuredWorkerArchive( +export async function captureStructuredWorkerArchive( identity: StructuredWorkerIdentity, agent: AgentType -): WorkerStructuredJournalArchive { - const page = readStructuredJournalPage(identity.sessionId) +): Promise { + // Opens a conversation at rest or one the idle sweep closed, so a resting worker's journal is + // still preserved. + const page = await readStructuredJournalPage(identity.sessionId) if (page) { return buildStructuredJournalArchive({ agent, @@ -232,10 +235,9 @@ export function captureStructuredWorkerArchive( // the worker's chat tab is a routine user action that does exactly that, so throwing there wedges // release on evidence that can never come and leaves `worker-abandon` as the only exit. // - // `exited` is the only verdict that qualifies: it needs a released lease WITH death evidence. - // `unverifiable` — no host installed, a lease handed to a TUI owner — means we could not look, - // and retaining is still right. - if (observeStructuredWorker(identity).status !== 'exited') { + // Only a worker this runtime no longer owns qualifies — released with its chat tab gone. An owned + // one, running or at rest, keeps its journal, and so does one we could not look at. + if (structuredWorkerOwned(identity.sessionId) !== false) { throw new OrchestrationError( 'archive_failed', 'Output could not be preserved for this structured worker; the session was retained.' diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts index 493f9d092c36..e50ac1a6fbb5 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' import { dispatchWriteFailureReason } from '../../../../shared/structured-agent-session-dispatch-rejection' const hostRef: { current: unknown } = { current: null } @@ -14,37 +15,34 @@ vi.mock('./structured-agent-session-create', () => ({ const { createStructuredWorkerSession, releaseStructuredWorkerSession, - sendStructuredWorkerPreamble, - structuredWorkerHoldId + sendStructuredWorkerPreamble } = await import('./orchestration-structured-worker-session') const { isUnknownWorkerStartOutcome } = await import('./orchestration/worker/worker-topology') const { structuredWorkerIdentities } = await import('../../structured-worker-identity') const { structuredWorkerChildIdentityEnv } = await import('../../structured-worker-child-identity-env') -function installHost() { - const hold = vi.fn(async () => {}) - const release = vi.fn() +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a host stub carrying only the members the worker start reaches. +function installHost(location = { executionHostId: 'local', wslDistro: null as string | null }) { const dispose = vi.fn() + const subscribe = vi.fn(async () => dispose) hostRef.current = { setSessionTabVisibility: async () => {}, close: async () => {}, deps: { store: { getRecord: () => ({ - location: { executionHostId: 'local', wslDistro: null }, + location, lease: { runtimeFence: 2, runtimeKind: 'native', claimStatus: 'live' } }) } }, - hold, - release, - subscribe: () => dispose + subscribe } - return { hold, release, dispose } + return { subscribe, dispose } } -describe('structured worker session hold', () => { +describe('structured worker session', () => { beforeEach(() => { structuredWorkerIdentities.clear() createSpy.mockReset() @@ -54,8 +52,8 @@ describe('structured worker session hold', () => { })) }) - it('takes a resume-capable hold at start and releases it only on settlement', async () => { - const { hold, release, dispose } = installHost() + it('binds only a redrive subscription at start, and settlement drops it', async () => { + const { subscribe, dispose } = installHost() const created = await createStructuredWorkerSession({ runtime: { ensureStructuredAgentSessionHost: async () => {} } as never, worktreeId: 'wt_1', @@ -63,18 +61,19 @@ describe('structured worker session hold', () => { dispatchId: 'd1', onJournalActivity: () => {} }) - // Without the hold, the release clock evicts the provider child 15s after a user closes the - // worker's chat tab, killing an idle worker mid-dispatch. - expect(hold).toHaveBeenCalledWith(created.identity.sessionId, structuredWorkerHoldId('d1')) - expect(release).not.toHaveBeenCalled() + // No hold: while the dispatch is open the idle sweep reads it from the orchestration database. + expect(hostRef.current).not.toHaveProperty('hold') + expect(subscribe).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: created.identity.sessionId }) + ) + expect(dispose).not.toHaveBeenCalled() releaseStructuredWorkerSession('d1') - expect(release).toHaveBeenCalledWith(created.identity.sessionId, structuredWorkerHoldId('d1')) expect(dispose).toHaveBeenCalledTimes(1) expect(structuredWorkerIdentities.get(created.identity.handle)).toBeNull() - // A second settlement is a no-op rather than a second release of the same holder. + // A second settlement is a no-op. releaseStructuredWorkerSession('d1') - expect(release).toHaveBeenCalledTimes(1) + expect(dispose).toHaveBeenCalledTimes(1) }) it('registers the identity BEFORE the session is created, so the child gets the handle', async () => { @@ -100,8 +99,8 @@ describe('structured worker session hold', () => { }) it('forgets the identity and discards the session when the start fails', async () => { - const { hold } = installHost() - hold.mockRejectedValueOnce(new Error('hold refused')) + // A session that resolves outside the local host is not a worker this runtime can own. + installHost({ executionHostId: 'local', wslDistro: 'Ubuntu' }) const closed: string[] = [] ;(hostRef.current as { close: (id: string) => Promise }).close = async (id) => { closed.push(id) @@ -114,7 +113,7 @@ describe('structured worker session hold', () => { dispatchId: 'd_fail', onJournalActivity: () => {} }) - ).rejects.toThrow('hold refused') + ).rejects.toThrow(/local execution host outside WSL/) // Neither a live provider child nor a registry entry may outlive the failed start. expect(closed).toHaveLength(1) expect(structuredWorkerIdentities.getBySessionId(closed[0]!)).toBeNull() @@ -128,7 +127,7 @@ describe('structured worker session hold', () => { } // `commit` answers this after `attach` SUCCEEDED and only the tab publish failed, so the // provider child is live. Reading it as "refused, nothing created" strands that child with no - // hold and no binding, and nothing else in the runtime ever retires it. + // binding, and nothing else in the runtime ever retires it. createSpy.mockImplementation(async () => ({ ok: false, refusal: { @@ -224,20 +223,69 @@ describe('structured worker session hold', () => { }) describe('structured worker dispatch preamble', () => { - function hostWithSubmission(submission: Record) { + type PreambleHost = Parameters[0]['host'] + type Settled = Pick + + function submissionOf(settled: Settled): AgentJournalSubmission { + return { + clientMessageId: 'c1', + fence: 7, + payloadFingerprint: 'fingerprint', + providerItemId: null, + submittedAt: 1, + resolvedAt: null, + ...settled + } + } + + function hostWithSubmission(submission: Settled, delivered?: Settled): PreambleHost { return { deps: { store: { getRecord: () => ({ lease: { runtimeFence: 7 } }) } }, - send: async () => ({ ok: true, value: { clientMessageId: 'c1', submission } }) - } as never + send: async () => ({ + ok: true, + replayed: false, + fence: 7, + cursor: { epoch: 'epoch-1', sequence: 1 }, + value: { clientMessageId: 'c1', submission: submissionOf(submission) } + }), + // What the submission settled as while the worker's agent started; undefined when the + // start outlasted the wait. + waitForSendSettlement: async () => + delivered + ? { + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { clientMessageId: 'c1', submission: submissionOf(delivered) } + } + : undefined + } } - const send = (host: never) => + const send = (host: PreambleHost) => sendStructuredWorkerPreamble({ host, sessionId: 's1', dispatchId: 'd1', preamble: 'spec' }) it('reports the preamble delivered only on an accepted submission', async () => { await expect( send(hostWithSubmission({ dispatchState: 'accepted', reason: null })) - ).resolves.toBeUndefined() + ).resolves.toBe('accepted') + }) + + it('waits for an accepted preamble to be delivered, and reports that delivery (W10)', async () => { + await expect( + send( + hostWithSubmission( + { dispatchState: 'pending', reason: null }, + { dispatchState: 'accepted', reason: null } + ) + ) + ).resolves.toBe('accepted') + }) + + it('reports a preamble still held for an agent that outlasted the wait, without failing the start (W10)', async () => { + // Held, not lost: the host delivers it when the agent starts. Throwing here tore the worker + // down, which rejected the preamble the start was about to deliver. + await expect( + send(hostWithSubmission({ dispatchState: 'pending', reason: null })) + ).resolves.toBe('pending') }) it('never claims delivery for a submission the provider never acknowledged', async () => { @@ -245,15 +293,13 @@ describe('structured worker dispatch preamble', () => { // into `unknown`, and `performSend` still returns ok. Reporting that as `dispatch_input: // accepted` marks the worker ready with no task, and the coordinator blocks in // `check --wait --types worker_done` until it times out. - for (const dispatchState of ['unknown', 'pending'] as const) { - const error = await send( - hostWithSubmission({ dispatchState, reason: 'provider child exited' }) - ).catch((thrown: unknown) => thrown) - expect((error as { code?: string }).code).toBe('operation_unknown') - // The wiring, not just the throw: this is the code that makes the start receipt - // `outcome_unknown` with the worker-show / worker-abandon recovery commands. - expect(isUnknownWorkerStartOutcome(error, 'dispatch_input')).toBe(true) - } + const error = await send( + hostWithSubmission({ dispatchState: 'unknown', reason: 'provider child exited' }) + ).catch((thrown: unknown) => thrown) + expect((error as { code?: string }).code).toBe('operation_unknown') + // The wiring, not just the throw: this is the code that makes the start receipt + // `outcome_unknown` with the worker-show / worker-abandon recovery commands. + expect(isUnknownWorkerStartOutcome(error, 'dispatch_input')).toBe(true) }) it('keeps a rejected preamble a proven failure under a code of its own', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts index 5bde461a0595..999445bf2d0b 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts @@ -1,19 +1,20 @@ /** - * Starting, holding and retiring a worker that IS a structured agent session. + * Starting and retiring a worker that IS a structured agent session. * * Three things make this different from the PTY worker path, and all three live here: * * - The session is created directly as structured, so readiness is the attach returning ok. There * is no boot-to-idle gap to wait on and no `tui-idle` edge to read. - * - A structured session's provider child is evicted 15s after its last HOLDER leaves, and holds - * come only from bound surfaces. A dispatched worker parked on mail is exactly that state, so - * the dispatch takes its own resume-capable hold and keeps it until the worker settles. + * - Nothing here keeps its agent running. The idle sweep leaves it running while its dispatch is + * open, reading that from the orchestration database; once the dispatch settles the agent rests + * like any chat's, and the next mail starts it. * - The dispatch preamble is a turn, not keystrokes. */ import { randomUUID } from 'node:crypto' import { isDefinitiveAgentSessionCreateRefusal } from '../../../../shared/agent-session-definitive-refusal' import type { AgentJournalMessageItem } from '../../../../shared/agent-session-journal-types' +import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../../shared/orchestration-timing-budgets' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { getStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import type { OrcaRuntimeService } from '../../orca-runtime' @@ -38,23 +39,16 @@ import { createStructuredAgentSessionForWorktree } from './structured-agent-sess type StructuredWorkerBinding = { sessionId: string handle: string - holderId: string disposeSubscription: () => void } const bindingsByDispatchId = new Map() -export function structuredWorkerHoldId(dispatchId: string): string { - return `orchestration:dispatch:${dispatchId}` -} - /** - * Drops the dispatch's hold, its redrive subscription and its parked mail; the release clock takes - * it from here. + * Drops the dispatch's redrive subscription, its registry entry and its parked mail. * - * EVERY settlement has to reach this — stop, release AND abandon. A surviving hold does not just - * leak: it keeps the provider child un-evictable for the life of the app, and makes host crash - * recovery respawn a child for a worker that was settled long ago. + * EVERY settlement has to reach this — stop, release AND abandon. A surviving subscription keeps + * nudging a session no dispatch owns. */ export function releaseStructuredWorkerSession( dispatchId: string, @@ -68,11 +62,6 @@ export function releaseStructuredWorkerSession( binding.disposeSubscription() structuredWorkerIdentities.forget(binding.handle) runtime?.forgetStructuredSessionMail?.(binding.sessionId) - try { - getStructuredAgentSessionHost()?.release(binding.sessionId, binding.holderId) - } catch (error) { - console.warn('[orchestration] structured worker hold release failed', dispatchId, error) - } } export async function createStructuredWorkerSession(args: { @@ -92,7 +81,7 @@ export async function createStructuredWorkerSession(args: { // resolves to whatever single leaf sits in the worktree — by default the COORDINATOR's pane. // // The scope is provisionally local; the record's own location is asserted local below, and a - // session that resolves anywhere else never reaches a hold. + // session that resolves anywhere else is discarded. const identity = structuredWorkerIdentities.register({ handle: mintStructuredWorkerHandle(), sessionId, @@ -141,13 +130,10 @@ export async function createStructuredWorkerSession(args: { 'A structured worker must run on the local execution host outside WSL.' ) } - const holderId = structuredWorkerHoldId(args.dispatchId) - await host.hold(sessionId, holderId) - const disposeSubscription = subscribeForRedrive(host, sessionId, args.onJournalActivity) + const disposeSubscription = await subscribeForRedrive(host, sessionId, args.onJournalActivity) bindingsByDispatchId.set(args.dispatchId, { sessionId, handle: identity.handle, - holderId, disposeSubscription }) return { identity, host } @@ -167,8 +153,8 @@ export async function createStructuredWorkerSession(args: { * * `ok` is not the test. `commit` answers `agent_session_operation_unknown` when `attach` SUCCEEDED * and only the tab publish failed, and a throw out of the commit half is past `attach` too — the - * pre-commit half never throws, it refuses. Both leave a live provider child that took no hold and - * has no binding, so nothing else in the runtime will ever retire it. Only a DEFINITIVE refusal + * pre-commit half never throws, it refuses. Both leave a session with a published tab and no + * binding, so nothing else in the runtime will ever retire it. Only a DEFINITIVE refusal * proves there is nothing to discard; everything else gets the best-effort close. */ function structuredCreateMayHaveCommitted( @@ -213,13 +199,22 @@ export async function discardStructuredWorkerSession( retireSettledStructuredWorkerTab(sessionId, runtime) } -/** Delivers the dispatch preamble as the worker's first turn. */ +/** What a preamble send reads of the host. */ +type StructuredWorkerPreambleHost = Pick< + StructuredAgentSessionHost, + 'send' | 'waitForSendSettlement' +> & { + deps: { store: { getRecord: (sessionId: string) => { lease: { runtimeFence: number } } | null } } +} + +/** Delivers the dispatch preamble as the worker's first turn. `pending`: the worker's agent had + * not taken it within the wait; the host still holds it for that agent, and never re-sends it. */ export async function sendStructuredWorkerPreamble(args: { - host: StructuredAgentSessionHost + host: StructuredWorkerPreambleHost sessionId: string dispatchId: string preamble: string -}): Promise { +}): Promise<'accepted' | 'pending'> { const body: AgentJournalMessageItem = { kind: 'message', role: 'user', @@ -244,9 +239,19 @@ export async function sendStructuredWorkerPreamble(args: { if (!result.ok) { throw new Error(`The dispatch preamble was refused: ${result.refusal.message}`) } - const submission = result.value.submission - if (submission.dispatchState === 'accepted') { - return + // Accepted is not delivered: the worker's agent may still be starting. + const submission = + result.value.submission.dispatchState === 'pending' + ? (( + await args.host + .waitForSendSettlement(args.sessionId, result.value.clientMessageId, { + budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS + }) + .catch(() => undefined) + )?.value.submission ?? result.value.submission) + : result.value.submission + if (submission.dispatchState === 'accepted' || submission.dispatchState === 'pending') { + return submission.dispatchState } if (submission.dispatchState === 'rejected') { // A rejection is a verdict, not a mystery: the preamble provably did not happen. @@ -304,17 +309,17 @@ const REDRIVE_MAX_WAIT_MS = 2_000 * idle worker — that is `deliverForHandle`, called when the message is enqueued and untouched * here. This is only the retry for mail already parked because the worker was busy. */ -function subscribeForRedrive( +async function subscribeForRedrive( host: StructuredAgentSessionHost, sessionId: string, onJournalActivity: (sessionId: string) => void -): () => void { +): Promise<() => void> { const coalescer = createKeyedTrailingEdgeCoalescer(onJournalActivity, { flushMs: REDRIVE_FLUSH_MS, maxWaitMs: REDRIVE_MAX_WAIT_MS }) try { - const unsubscribe = host.subscribe({ + const unsubscribe = await host.subscribe({ id: `orchestration:redrive:${sessionId}`, sessionId, emit: (event) => { @@ -324,7 +329,7 @@ function subscribeForRedrive( } }) // Disposal drops the pending timer rather than flushing it: every settlement reaches here, and - // a redrive that fires after the hold is gone would nudge a session no dispatch owns. + // a redrive that fires after it would nudge a session no dispatch owns. return () => { coalescer.dispose() unsubscribe() diff --git a/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts index aa8c7c58574f..199e43fae7b3 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts @@ -42,6 +42,7 @@ vi.mock('./orchestration-structured-worker-session', async (importOriginal) => ( ...(await importOriginal>()), sendStructuredWorkerPreamble: async (args: { preamble: string }) => { structuredPreambles.push(args.preamble) + return 'accepted' }, releaseStructuredWorkerSession: () => {}, discardStructuredWorkerSession: async () => {} @@ -253,10 +254,14 @@ describe('a worker cannot tell which mode it is running in', () => { source: 'terminal' }) - expect(read).toThrow(/has no terminal output/) + const refusal = await read().then( + () => '', + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ) + expect(refusal).toMatch(/has no terminal output/) // The refusal names a source that works instead of naming the worker's kind. - expect(read).toThrow(/--source auto or --source transcript/) - expect(read).not.toThrow(/structured/i) + expect(refusal).toMatch(/--source auto or --source transcript/) + expect(refusal).not.toMatch(/structured/i) }) it('never claims a structured worker was checked for a human-answerable prompt', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts index 52cf3579016c..a2beea9ea05c 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts @@ -30,7 +30,7 @@ vi.mock('./orchestration/federation/federated-worker-start', () => ({ })) vi.mock('./orchestration-structured-worker-session', async (importOriginal) => ({ ...(await importOriginal>()), - sendStructuredWorkerPreamble: async () => {}, + sendStructuredWorkerPreamble: async () => 'accepted', releaseStructuredWorkerSession: () => {}, discardStructuredWorkerSession: async () => {} })) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-group.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-group.test.ts index b076be065931..62d2b5da4950 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-group.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-group.test.ts @@ -69,7 +69,7 @@ describe('orchestration.send group addresses', () => { return terminal ? `${terminal.tabId}:${terminal.leafId}` : null }) vi.spyOn(runtime, 'getAgentStatusForHandle').mockImplementation( - (handle: string) => agentStatuses?.[handle] ?? null + async (handle: string) => agentStatuses?.[handle] ?? null ) } diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-group.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-group.ts index cb75bca27054..ebcff6c284e5 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-group.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-group.ts @@ -148,7 +148,7 @@ export async function sendGroupMessage(args: { const { terminals } = await runtime.listTerminals(undefined, undefined, { includeVisualLayouts: false }) - agents = [...terminals, ...listAddressableStructuredWorkers()] + agents = [...terminals, ...listAddressableStructuredWorkers(db)] } // Revalidate after discovery before selecting recipients or writing mail. revalidateLegacyCoordinator?.() @@ -167,8 +167,23 @@ export async function sendGroupMessage(args: { agents, warnings: groupWarnings }) - const handles = resolveGroupAddress(groupAddress, from, candidates, (handle: string) => - runtime.getAgentStatusForHandle(handle) + // Read up front: a structured worker's status comes from its journal, which may need opening. + const statuses = + groupAddress.toLowerCase() === '@idle' + ? new Map( + await Promise.all( + candidates.map( + async (candidate) => + [candidate.handle, await runtime.getAgentStatusForHandle(candidate.handle)] as const + ) + ) + ) + : new Map() + const handles = resolveGroupAddress( + groupAddress, + from, + candidates, + (handle: string) => statuses.get(handle) ?? null ) if (handles.length === 0) { // Preserve the recovery addresses even when every worker was skipped. diff --git a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts index ecb3270874e4..bc2b9601cebc 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts @@ -5,6 +5,7 @@ import { dispatchPreambleSendOptions } from '../../../../orchestration/preamble' import { sendStructuredWorkerPreamble } from '../../orchestration-structured-worker-session' +import type { WorkerTurnStartObservation } from './worker-start-turn-observation' import type { createStructuredWorkerSessionForWorktree } from './worker-topology' type StructuredSession = Awaited> | null @@ -14,7 +15,8 @@ type StructuredSession = Awaited { +}): Promise<{ + prompt?: RuntimeTerminalSend['prompt'] + structuredTurnStart?: WorkerTurnStartObservation +}> { const { runtime, structuredSession, terminalHandle } = args const preamble = buildDispatchPreamble({ // Depth only. A worker is taught the same verbs whichever mode it runs in, so this must not @@ -45,19 +50,32 @@ export async function deliverWorkerDispatchPreamble(args: { cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) }) if (structuredSession) { - await sendStructuredWorkerPreamble({ + const delivery = await sendStructuredWorkerPreamble({ host: structuredSession.host, sessionId: structuredSession.identity.sessionId, dispatchId: args.dispatchId, preamble }) - return undefined + return { + structuredTurnStart: + delivery === 'accepted' + ? { verdict: 'observed' } + : { + verdict: 'unobserved', + reason: + 'The dispatch preamble was accepted, but the agent had not started to take it. It ' + + 'is delivered when the agent starts; if the worker then reports, this Dispatch ' + + 'settles normally.' + } + } + } + return { + prompt: ( + await runtime.sendTerminalAgentPrompt( + terminalHandle, + preamble, + dispatchPreambleSendOptions(args.requestId) + ) + ).prompt } - return ( - await runtime.sendTerminalAgentPrompt( - terminalHandle, - preamble, - dispatchPreambleSendOptions(args.requestId) - ) - ).prompt } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/failed-worker-start-teardown.ts b/src/main/runtime/rpc/methods/orchestration/worker/failed-worker-start-teardown.ts index 250b639fc603..758bc3397cb7 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/failed-worker-start-teardown.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/failed-worker-start-teardown.ts @@ -8,10 +8,10 @@ import type { createStructuredWorkerSessionForWorktree } from './worker-topology /** * Undoes what a start created before it failed. * - * A start that never reached ready leaves no settlement to release the hold later, and its session - * was already published as a chat tab — without the discard, a failed start strands a dead chat tab - * that the durable restore index republishes on every app launch. Both halves are best-effort by - * construction, so neither can replace the real error. + * A start that never reached ready leaves no settlement to release its binding later, and its + * session was already published as a chat tab — without the discard, a failed start strands a dead + * chat tab that the durable restore index republishes on every app launch. Both halves are + * best-effort by construction, so neither can replace the real error. * * A created PTY terminal is deliberately NOT torn down: its custody row was written at creation, so * `worker-release` on the failed Dispatch owns that cleanup and the coordinator decides when. diff --git a/src/main/runtime/rpc/methods/orchestration/worker/structured-worker-release-host.ts b/src/main/runtime/rpc/methods/orchestration/worker/structured-worker-release-host.ts new file mode 100644 index 000000000000..4ec457f962d6 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/structured-worker-release-host.ts @@ -0,0 +1,33 @@ +import { getStructuredAgentSessionHost } from '../../../../../native-chat/agent-session-wire/structured-agent-session-registry' +import type { OrcaRuntimeService } from '../../../../orca-runtime' + +/** + * The structured host a worker release reads, installed and with its restart leases settled. + * + * Observation and archive capture both read the structured host, and after a restart nothing has + * installed it yet — the startup recovery reconciler runs exactly this path. Installing it here is + * what lets the release see the session instead of reporting it unreadable. Startup no longer + * reconciles ahead of this pass either, and an unreconciled worker that ended with the previous app + * run still reads as live, so the release waits for that reconcile too. + * + * NOT yet handled, and deliberately follow-up: rebinding a restarted runtime to a structured + * worker's redrive subscription. Until that exists, a worker that survives a restart has its + * parked mail wait for the next arrival rather than a settle edge. + */ +export async function prepareStructuredHostForWorkerRelease( + runtime: Pick, + dispatchId: string +): Promise { + await runtime.ensureStructuredAgentSessionHost().catch((error: unknown) => { + console.warn('[orchestration] structured host install failed before release', dispatchId, error) + }) + await getStructuredAgentSessionHost() + ?.reconcileRestartLeases() + .catch((error: unknown) => { + console.warn( + '[orchestration] structured lease reconcile failed before release', + dispatchId, + error + ) + }) +} diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts index 5e8babf3c5e1..3187d55cccfc 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts @@ -143,7 +143,7 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS = [ `Worker Dispatch ${params.dispatch} no longer resolves to its exact process.` ) } - const structured = readStructuredWorkerOutput({ + const structured = await readStructuredWorkerOutput({ db, dispatchId: params.dispatch, workerState: worker?.state ?? 'unsupervised', @@ -200,9 +200,8 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS = [ const abandoned = runtime.getOrchestrationDb().abandonWorkerDispatch(params.dispatch) if (abandoned.disposition === 'context_only') { if (!abandoned.alreadySettled) { - // Abandon settles the Dispatch, so it owes the same hold release stop and release do. - // A surviving hold pins the provider child for the life of the app and makes host crash - // recovery respawn a worker nobody is waiting on. + // Abandon settles the Dispatch, so it owes the same binding release stop and release do: + // a surviving redrive subscription keeps nudging a worker nobody is waiting on. releaseStructuredWorkerSession(params.dispatch, runtime) runtime.notifyMessageArrived(`dispatch:${params.dispatch}`, 'status') } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-observation.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-observation.ts index 83b3d020f4ca..81f3a2c2975a 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-observation.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-observation.ts @@ -9,6 +9,7 @@ import { observeStructuredWorker, resolveStructuredWorkerForDispatch } from '../../orchestration-structured-worker-lifecycle' +import { structuredWorkerAddressable } from '../../../../structured-worker-custody' import type { DispatchContextRow, FederatedDispatchRow, @@ -27,6 +28,9 @@ export async function inspectWorkerTerminal( reason?: string /** Set only on a proven-exact worker parked on a prompt that needs a human. */ agentWait?: RuntimeTerminalInteractiveWait | null + /** Structured workers only: whether mail still reaches it — at rest included, since the mail + * starts it. Absent when ownership cannot be read. `status` stays the process verdict. */ + addressable?: boolean }> { const worker = db.getWorkerDispatch(dispatchId) const terminalHandle = @@ -49,11 +53,17 @@ export async function inspectWorkerTerminal( processIncarnation: structured.processIncarnation }) const observation = observeStructuredWorker(structured) + const addressable = structuredWorkerAddressable( + db, + structured.sessionId, + db.getWorkerTerminalResourceByHandle?.(structured.handle) + ) return { terminal: null, exact, status: exact ? observation.status : 'identity_changed', - ...(exact && observation.reason ? { reason: observation.reason } : {}) + ...(exact && observation.reason ? { reason: observation.reason } : {}), + ...(exact && addressable !== null ? { addressable } : {}) } } const terminal = await runtime.showTerminal(terminalHandle).catch(() => null) @@ -119,7 +129,8 @@ export function exposeObservation(observation: Awaited { const { runtime, db, dispatchId, resource } = args if (isStructuredWorkerHandle(resource.terminal_handle)) { - // Observation and archive capture both read the structured host, and after a restart nothing - // has installed it yet — the startup recovery reconciler runs exactly this path. Installing it - // here is what lets the release see the session instead of reporting it unreadable. - // - // NOT yet handled, and deliberately follow-up: rebinding a restarted runtime to a structured - // worker's hold and redrive subscription. Until that exists, a worker that survives a restart - // keeps no hold, so its child is evictable and its parked mail waits for the next arrival - // rather than a settle edge. - await runtime.ensureStructuredAgentSessionHost().catch((error: unknown) => { - console.warn( - '[orchestration] structured host install failed before release', - dispatchId, - error - ) - }) + await prepareStructuredHostForWorkerRelease(runtime, dispatchId) } const worker = db.getWorkerDispatch(dispatchId) if (!worker || worker.agent_terminal_handle !== resource.terminal_handle) { diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts new file mode 100644 index 000000000000..359d27c30502 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { setStructuredAgentSessionHost } from '../../../../../native-chat/agent-session-wire/structured-agent-session-registry' +import { mintStructuredWorkerHandle } from '../../../../structured-worker-identity' + +const { order } = vi.hoisted(() => { + const order: string[] = [] + return { order } +}) + +vi.mock('./worker-observation', () => ({ + inspectWorkerTerminal: vi.fn(async () => { + order.push('observe') + return { status: 'identity_changed' } + }) +})) + +const { completeWorkerTerminalRelease } = await import('./worker-release-completion') + +afterEach(() => { + order.length = 0 + setStructuredAgentSessionHost(null) +}) + +describe('structured worker release after a restart', () => { + it('reads the worker only once the restart reconcile has settled its lease', async () => { + const handle = mintStructuredWorkerHandle() + const runtime = { + ensureStructuredAgentSessionHost: vi.fn(async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the release reads only the reconcile stubbed here. + setStructuredAgentSessionHost({ + reconcileRestartLeases: async () => { + // A reconcile that probes processes settles after a macrotask, not a microtask. + await new Promise((resolve) => setTimeout(resolve, 0)) + order.push('reconcile') + } + } as never) + }) + } + const db = { + getWorkerDispatch: () => ({ agent_terminal_handle: handle }), + revertWorkerTerminalReleaseToRetained: () => ({ archive_source: null, archive_status: null }), + recordWorkerTerminalRecoveryAttempt: vi.fn() + } + + const receipt = await completeWorkerTerminalRelease({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this path reads only the runtime member stubbed above. + runtime: runtime as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retained path reads only the dispatch rows stubbed above. + db: db as never, + dispatchId: 'dispatch-1', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retained path reads only the id and handle. + resource: { id: 'resource-1', terminal_handle: handle } as never, + mode: 'recovery' + }) + + expect(receipt.state).toBe('retained') + expect(order).toEqual(['reconcile', 'observe']) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts new file mode 100644 index 000000000000..e39a06110dcd --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('../../../../orchestration/preamble', () => ({ buildDispatchPreamble: () => 'preamble' })) +vi.mock('./worker-topology', async (importOriginal) => ({ + ...(await importOriginal>()), + monitorWorkerSetup: () => {} +})) + +const { deliverAndSettleWorkerStartReadiness } = await import('./worker-start-readiness-settlement') + +function settle(delivered: 'accepted' | undefined) { + const db = { + getWorkerDispatch: () => ({ state: 'starting' }), + markWorkerStartUnknown: vi.fn(() => ({ + stage: 'turn_start_unobserved', + residual_resources: '[]' + })), + markWorkerDispatchReady: vi.fn(() => ({ state: 'ready', stage: 'ready' })) + } + const host = { + deps: { store: { getRecord: () => ({ lease: { runtimeFence: 1 } }) } }, + send: async () => ({ + ok: true, + value: { clientMessageId: 'c1', submission: { dispatchState: 'pending', reason: null } } + }), + // undefined: the worker's agent was still starting when the wait ran out. + waitForSendSettlement: async () => + delivered + ? { value: { clientMessageId: 'c1', submission: { dispatchState: delivered } } } + : undefined + } + const args = { + runtime: { + getNestedWorkerMaxDepth: () => 3, + getTerminalOrchestrationCliCommand: () => 'orca' + }, + db, + run: { id: 'run_1' }, + task: { id: 't1', spec: 'do the thing' }, + dispatchId: 'd1', + dispatchDepth: 0, + structuredSession: { host, identity: { sessionId: 's1' } }, + terminalHandle: 'structured_worker_1', + coordinatorHandle: 'term_c', + dispatchCapability: 'capability', + devMode: undefined, + requestId: 'r1', + agent: 'claude', + setupReceipt: {}, + launchReceipt: {}, + mode: {}, + timeoutMs: 60_000, + effects: [], + terminalRevealWarning: undefined, + onStage: () => {} + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fakes implement exactly the runtime, db and host members this settlement reaches. + const receipt = deliverAndSettleWorkerStartReadiness(args as never) + return { db, receipt } +} + +describe('a structured worker whose agent outlasts the preamble wait', () => { + it('parks as start-unknown instead of failing the start, and never names a screen to read', async () => { + const { db, receipt } = settle(undefined) + + // Resolving, not throwing, is what keeps the worker's session: a throw tears it down. + await expect(receipt).resolves.toMatchObject({ + state: 'outcome_unknown', + turnStart: 'unobserved', + nextCommands: [ + 'orca orchestration worker-show --dispatch d1 --json', + 'orca orchestration worker-abandon --dispatch d1 --json' + ] + }) + expect(db.markWorkerStartUnknown).toHaveBeenCalledWith( + 'd1', + 'turn_start_unobserved', + expect.stringContaining('delivered when the agent starts'), + expect.anything() + ) + expect(db.markWorkerDispatchReady).not.toHaveBeenCalled() + }) + + it('is ready once the agent took the preamble within the wait', async () => { + const { db, receipt } = settle('accepted') + + await expect(receipt).resolves.toMatchObject({ state: 'ready', turnStart: 'observed' }) + expect(db.markWorkerStartUnknown).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts index a3c57a357d27..7e9118daffb5 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts @@ -47,7 +47,7 @@ export async function deliverAndSettleWorkerStartReadiness(args: { const { runtime, db, run, task, structuredSession, terminalHandle, effects } = args args.onStage('dispatch_input') - const promptDelivery = await deliverWorkerDispatchPreamble({ + const delivery = await deliverWorkerDispatchPreamble({ runtime, structuredSession, terminalHandle, @@ -71,11 +71,11 @@ export async function deliverAndSettleWorkerStartReadiness(args: { // The write above was accepted without waiting on provider hooks; now demand the positive // evidence the receipt claims is observable. A worker whose turn never starts must not be // reported ready — a wedged agent and a working one looked identical before this gate. - // A structured preamble send is acknowledged by the provider or throws, so it is already - // positive evidence. - const turnStart: WorkerTurnStartObservation = structuredSession - ? { verdict: 'observed' } - : await observeWorkerTurnStart({ runtime, terminalHandle, prompt: promptDelivery }) + // A structured preamble send is its own evidence: acknowledged, or still held for its agent. + const promptDelivery = delivery.prompt + const turnStart: WorkerTurnStartObservation = + delivery.structuredTurnStart ?? + (await observeWorkerTurnStart({ runtime, terminalHandle, prompt: promptDelivery })) const deliveredPrompt = turnStart.prompt ?? promptDelivery monitorWorkerSetup({ runtime, @@ -98,7 +98,7 @@ export async function deliverAndSettleWorkerStartReadiness(args: { id: terminalHandle, state: 'turn_unobserved' }) - const reason = describeUnobservedWorkerTurnStart(args.agent) + const reason = turnStart.reason ?? describeUnobservedWorkerTurnStart(args.agent) const worker = db.markWorkerStartUnknown( args.dispatchId, 'turn_start_unobserved', @@ -122,7 +122,8 @@ export async function deliverAndSettleWorkerStartReadiness(args: { residualResources: JSON.parse(worker.residual_resources) as unknown[], nextCommands: [ `orca orchestration worker-show --dispatch ${args.dispatchId} --json`, - `orca terminal read --terminal ${terminalHandle} --screen`, + // A structured worker has no screen to read. + ...(structuredSession ? [] : [`orca terminal read --terminal ${terminalHandle} --screen`]), `orca orchestration worker-abandon --dispatch ${args.dispatchId} --json` ], ...(args.terminalRevealWarning ? { warning: args.terminalRevealWarning } : {}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts index d1c9e59adfa5..cc648ee7ddeb 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts @@ -19,6 +19,8 @@ export type WorkerTurnStartVerdict = 'observed' | 'permission' | 'unsupported' | export type WorkerTurnStartObservation = { verdict: WorkerTurnStartVerdict prompt?: RuntimeTerminalPromptDelivery + /** Why an `unobserved` start is unknown, when the default PTY wording does not fit. */ + reason?: string } function classifyPromptDelivery(prompt: RuntimeTerminalPromptDelivery): WorkerTurnStartVerdict { diff --git a/src/main/runtime/rpc/methods/session-tabs-listing-order.test.ts b/src/main/runtime/rpc/methods/session-tabs-listing-order.test.ts new file mode 100644 index 000000000000..0ad7322fd13f --- /dev/null +++ b/src/main/runtime/rpc/methods/session-tabs-listing-order.test.ts @@ -0,0 +1,113 @@ +// Every host answer that carries chat tabs waits for the host's one-shot tab publication, so no +// client can apply a snapshot that lacks a listed chat and drop its saved tab (F15b). + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' +import { OrcaRuntimeService } from '../../orca-runtime' +import type { RpcRequest } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { SESSION_TAB_METHODS } from './session-tabs' + +afterEach(() => setStructuredAgentSessionHost(null)) + +type RestoreInternals = { + getClientSettings(): { experimentalStructuredNativeChat: boolean } + supportsAuthoritativeSessionTabsInventory(): boolean + hasPersistedStructuredAgentSessionStore(): boolean + getKnownWorkspaceSessionWorktreeIds(): Set + hydrateHeadlessMobileSessionTabsFromWorkspaceSession(): Set + refreshMobileSessionPtyRecords(): Promise | null> + ensureStructuredAgentSessionHost(): Promise +} + +/** A restarted runtime whose host install is held until `install.resolve()`. */ +function restartedRuntime() { + const install = Promise.withResolvers() + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as RestoreInternals + internal.getClientSettings = () => ({ experimentalStructuredNativeChat: true }) + // The PTY census behind an authoritative inventory is not what this pins. + internal.supportsAuthoritativeSessionTabsInventory = () => false + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.getKnownWorkspaceSessionWorktreeIds = () => new Set() + internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() + internal.refreshMobileSessionPtyRecords = async () => new Set() + internal.ensureStructuredAgentSessionHost = async () => { + await install.promise + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the restore reads only these host members. + setStructuredAgentSessionHost({ + reconcileRestartLeases: async () => undefined, + restoreStartupSessions: async () => undefined, + listVisibleSessionIds: () => ['listed-chat'], + listPersistedSessionTabs: () => [ + { sessionId: 'listed-chat', workspaceId: 'workspace-1', agent: 'codex' } + ] + } as never) + } + return { runtime, install } +} + +const request = (method: string, params?: unknown): RpcRequest => ({ + id: `req-${method}`, + authToken: 'tok', + method, + params +}) + +async function settle(): Promise { + for (let turn = 0; turn < 20; turn += 1) { + await Promise.resolve() + } +} + +describe('session tab answers after a restart', () => { + it('holds list, listAll, subscribe and subscribeAll until the listed chats are published', async () => { + const { runtime, install } = restartedRuntime() + const dispatcher = new RpcDispatcher({ runtime, methods: SESSION_TAB_METHODS }) + const context = { + clientKind: 'runtime' as const, + clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + } + const answers = new Map() + const call = (method: string, params?: unknown) => { + const messages: string[] = [] + answers.set(method, messages) + return dispatcher.dispatchStreaming( + request(method, params), + (message) => messages.push(message), + context + ) + } + + const calls = [ + call('session.tabs.list', { worktree: 'id:workspace-1' }), + call('session.tabs.listAll'), + call('session.tabs.subscribe', { worktree: 'id:workspace-1' }), + call('session.tabs.subscribeAll') + ] + await settle() + for (const [method, messages] of answers) { + expect({ method, messages }).toEqual({ method, messages: [] }) + } + + install.resolve() + await vi.waitFor( + () => { + for (const [method, messages] of answers) { + expect({ method, answered: messages.length > 0 }).toEqual({ method, answered: true }) + } + }, + { timeout: 10_000 } + ) + for (const [method, messages] of answers) { + expect({ method, carries: messages[0]?.includes('agent-session:listed-chat') }).toEqual({ + method, + carries: true + }) + } + runtime.cleanupSubscriptionsByPrefix('session.tabs:') + await Promise.allSettled(calls) + }) +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-admission.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-admission.test.ts index de62b6b5b528..554c0912ff03 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-admission.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-admission.test.ts @@ -32,16 +32,16 @@ describe('admission revoked while a session is still open', () => { it.each(CLEANUP_METHODS)( 'still serves $method after the host setting is turned off', - async ({ method, params, hostCall }) => { - const response = await call(method, params, STRUCTURED_CLIENT, SETTING_OFF) + async (entry) => { + const response = await call(entry.method, entry.params, STRUCTURED_CLIENT, SETTING_OFF) expect(response).toMatchObject({ ok: true }) - // `unsubscribe` retires runtime-owned subscriptions rather than calling the host, so its - // result payload is the observable effect. - if (hostCall === 'unsubscribe') { - expect(response).toMatchObject({ result: { unsubscribed: true } }) + // `unsubscribe` retires runtime-owned subscriptions and `release` is a no-op, so neither + // calls the host: the result payload is the observable effect. + if (entry.hostCall === null) { + expect(response).toMatchObject({ result: entry.result }) } else { - expect(hostCalls[hostCall]).toHaveBeenCalled() + expect(hostCalls[entry.hostCall]).toHaveBeenCalled() } } ) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts new file mode 100644 index 000000000000..c2633ffe3985 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts @@ -0,0 +1,378 @@ +// A chat at rest, through the RPC surface a client actually calls: opening it starts nothing, what +// it can answer without an agent it answers, and the first send is what starts one. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../../shared/agent-session-mutation-envelope' +import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { openJournalDatabase } from '../../../native-chat/agent-session-journal/journal-database' +import { + journalDatabaseFile, + journalDirectoryFor +} from '../../../native-chat/agent-session-journal/journal-paths' +import { + HOST_TEST_LOCATION, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId +} from '../../../native-chat/agent-session-wire/structured-agent-session-host-test-data' +import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' +import { + createRestTestRig, + foundRestTestChat, + REST_TEST_CALLER as CALLER, + REST_TEST_SESSION as SESSION, + REST_TEST_THREAD, + restTestSend, + type RestTestRig +} from '../../../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' +import { OrcaRuntimeService } from '../../orca-runtime' +import type { RpcResponse } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { closeStructuredAgentSessionChild } from '../../structured-agent-session-close' +import { discardStructuredWorkerSession } from './orchestration-structured-worker-session' +import { STRUCTURED_AGENT_SESSION_METHODS } from './structured-agent-session' + +const CLIENT = { + clientId: 'device-1', + clientKind: 'runtime' as const, + clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY], + connectionId: 'connection-1' +} + +let rig: RestTestRig +let dispatcher: RpcDispatcher +let requests = 0 + +async function call(method: string, params: unknown): Promise { + const replies: RpcResponse[] = [] + requests += 1 + await dispatcher.dispatchStreaming( + { id: `request-${requests}`, authToken: 'token', method, params }, + (raw) => replies.push(JSON.parse(raw) as RpcResponse), + CLIENT + ) + return replies +} + +/** A chat that once ran, reopened by a fresh host: nothing of it is in memory. */ +async function restingChat(): Promise { + await foundRestTestChat(rig) + await rig.store.replaceSessionOptions({ + sessionId: SESSION, + fence: rig.store.getRecord(SESSION)!.lease.runtimeFence, + options: { model: 'gpt-saved', effort: 'high' }, + now: rig.clock.now + }) + await rig.restart() + setStructuredAgentSessionHost(rig.host) + rig.adapter.acquire.mockClear() + rig.adapter.dispatch.mockClear() + rig.adapter.readOptions.mockClear() +} + +beforeEach(async () => { + requests = 0 + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) + setStructuredAgentSessionHost(rig.host) + const runtime = new OrcaRuntimeService() + vi.spyOn(runtime, 'getClientSettings').mockImplementation( + () => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the RPC gate reads only this one setting. + ({ experimentalStructuredNativeChat: true }) as ReturnType< + OrcaRuntimeService['getClientSettings'] + > + ) + dispatcher = new RpcDispatcher({ runtime, methods: STRUCTURED_AGENT_SESSION_METHODS }) +}) + +afterEach(async () => { + setStructuredAgentSessionHost(null) + await rig.dispose() +}) + +describe('opening a chat at rest (P2-01)', () => { + it('reads, subscribes and answers everything without starting an agent', async () => { + await restingChat() + expect(rig.host.hasSession(SESSION)).toBe(false) + + const [history] = await call('agentSession.history', { sessionId: SESSION, direction: 'tail' }) + const frames = await call('agentSession.subscribe', { sessionId: SESSION }) + const [options] = await call('agentSession.options', { sessionId: SESSION }) + const [commands] = await call('agentSession.commands', { sessionId: SESSION }) + const [outline] = await call('agentSession.conversationOutline', { sessionId: SESSION }) + const [status] = await call('agentSession.handoffStatus', { sessionId: SESSION }) + const [held] = await call('agentSession.hold', { sessionId: SESSION, holderId: 'pane' }) + + expect(rig.adapter.acquire).not.toHaveBeenCalled() + expect(history).toMatchObject({ ok: true, result: { ok: true } }) + const snapshot = frames.find((frame) => frame.ok && 'result' in frame) + expect(snapshot).toMatchObject({ result: { type: 'snapshot', commands: null } }) + expect(options).toMatchObject({ + ok: true, + result: { current: { model: 'gpt-saved', effort: 'high' } } + }) + // Absent, never an empty list: the composer keeps its own menu. + expect(commands).toMatchObject({ ok: true, result: {} }) + expect(commands?.ok && 'result' in commands && commands.result).not.toHaveProperty( + 'commands', + [] + ) + expect(outline).toMatchObject({ ok: true }) + expect(status).toMatchObject({ ok: true }) + expect(held).toMatchObject({ ok: true, result: { held: true } }) + }) + + // Worktree activation asks this for every chat tab in the worktree. + it('answers the owner check from the record without opening the chat', async () => { + await restingChat() + + const [status] = await call('agentSession.handoffStatus', { sessionId: SESSION }) + + expect(status).toMatchObject({ ok: true, result: { owner: expect.any(String) } }) + expect(rig.host.hasSession(SESSION)).toBe(false) + }) + + it('starts the agent on the first send (P2-01)', async () => { + await restingChat() + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + const sent = await rig.host.send(CALLER, restTestSend('wake up', fence)) + if (!sent.ok) { + throw new Error(`send refused: ${sent.refusal.code}`) + } + // Awaits the provider's answer itself rather than polling for it, however slow the start. + await rig.host.waitForSendSettlement(SESSION, sent.value.clientMessageId) + expect(rig.adapter.acquire).toHaveBeenCalledOnce() + expect(rig.adapter.dispatch).toHaveBeenCalledOnce() + }) +}) + +describe('the accessor', () => { + it('opens a closed conversation once, however many readers arrive together (P2-02)', async () => { + // Written by the provider alone, so nothing but the readers below ever opens it here. + const attached = await rig.host.attach(CALLER, hostTestAttachParams(null)) + expect(attached.ok).toBe(true) + rig.adapter.acquire.mock.calls + .at(-1)?.[0] + .events?.appendItem( + { provider: 'codex', threadId: REST_TEST_THREAD, turnId: 'turn-1', ordinal: 1 }, + hostTestMessage('from the provider') + ) + await rig.host.flushStreamedEvents(SESSION) + await rig.restart() + setStructuredAgentSessionHost(rig.host) + rig.adapter.acquire.mockClear() + const open = vi.spyOn(rig.host.collaboratorsForTests().conversationDelivery, 'open') + await Promise.all([ + ...Array.from({ length: 5 }, () => + call('agentSession.history', { sessionId: SESSION, direction: 'tail' }) + ), + call('agentSession.subscribe', { sessionId: SESSION }) + ]) + const openedJournals = new Set( + await Promise.all(open.mock.results.map((result) => result.value)) + ) + expect(openedJournals.size).toBe(1) + expect(rig.adapter.acquire).not.toHaveBeenCalled() + }) + + it('opens nothing once quit began, for a read that was already waiting on the lock', async () => { + await restingChat() + // The host's per-session queue, held so the read waits behind it. + const { serialize, lifetime } = rig.host.collaboratorsForTests() + let release = (): void => undefined + const held = new Promise((started) => { + void serialize(SESSION, () => { + started() + return new Promise((resolve) => (release = resolve)) + }) + }) + await held + const read = rig.host.history({ sessionId: SESSION, direction: 'tail' }) + + // Disposed as quit's first teardown step does. + lifetime.dispose() + release() + + await expect(read).rejects.toThrow() + expect(rig.host.hasSession(SESSION)).toBe(false) + }) + + it('opens a corrupt journal through the recovering open and still accepts a send (P2-03)', async () => { + await foundRestTestChat(rig) + await rig.host.flushAllStreamedEvents() + const directory = journalDirectoryFor(rig.root, { + workspaceId: HOST_TEST_LOCATION.workspaceId, + sessionId: SESSION + }) + // A row that no longer parses: the recovering open keeps the readable prefix and rebuilds. + const opened = openJournalDatabase(journalDatabaseFile(directory)) + try { + opened.db.prepare('UPDATE journal_rows SET row_json = ? WHERE seq = ?').run('}{', 2) + } finally { + opened.db.close() + } + await rig.restart() + setStructuredAgentSessionHost(rig.host) + + const frames = await call('agentSession.subscribe', { sessionId: SESSION }) + expect(frames.some((frame) => !frame.ok)).toBe(false) + expect(frames.find((frame) => frame.ok)).toMatchObject({ result: { type: 'snapshot' } }) + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + expect((await rig.host.send(CALLER, restTestSend('after the repair', fence))).ok).toBe(true) + }) + + it('subscribes an old mobile client that holds first even when no agent can start (P2-06)', async () => { + await restingChat() + rig.adapter.acquire.mockRejectedValue(new Error('auth expired')) + + const [held] = await call('agentSession.hold', { sessionId: SESSION, holderId: 'mobile' }) + const frames = await call('agentSession.subscribe', { sessionId: SESSION }) + expect(held).toMatchObject({ ok: true }) + expect(frames.find((frame) => frame.ok)).toMatchObject({ result: { type: 'snapshot' } }) + expect(rig.adapter.acquire).not.toHaveBeenCalled() + }) +}) + +describe('options at rest', () => { + it('records a pick as intent and replays it at the next start (P2-16)', async () => { + await restingChat() + const fields = { key: 'model', value: 'gpt-picked' } + const [picked] = await call('agentSession.setOption', { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: rig.store.getRecord(SESSION)!.lease.runtimeFence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.setOption', + sessionId: SESSION, + fields + }) + }, + ...fields + }) + expect(picked).toMatchObject({ ok: true, result: { ok: true } }) + expect(rig.store.getRecord(SESSION)?.options).toMatchObject({ model: 'gpt-picked' }) + expect(rig.adapter.acquire).not.toHaveBeenCalled() + + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + await rig.host.send(CALLER, restTestSend('use the new model', fence)) + await vi.waitFor(() => expect(rig.adapter.acquire).toHaveBeenCalledOnce()) + expect(rig.adapter.acquire.mock.calls[0]?.[0]).toMatchObject({ + options: expect.objectContaining({ model: 'gpt-picked' }) + }) + }) + + it('answers the provider-level features of a chat at rest (P2-17)', async () => { + await restingChat() + Object.assign(rig.host.deps.adapter, { + supportsThreadGoal: (_id: string, agent?: string) => agent === 'codex', + recordsContextUsage: (_id: string, agent?: string) => agent === 'claude', + rewindSupport: (_id: string, agent?: string) => + agent === 'codex' ? { supported: true } : { supported: false, reason: 'unsupported' } + }) + const [options] = await call('agentSession.options', { sessionId: SESSION }) + expect(options).toMatchObject({ + ok: true, + result: { threadGoal: { current: null }, rewind: { supported: true } } + }) + expect(rig.adapter.acquire).not.toHaveBeenCalled() + expect(rig.adapter.readOptions).not.toHaveBeenCalled() + }) +}) + +describe('an agent exit', () => { + it('is shown, not respawned; the next send starts the agent (P2-18)', async () => { + await foundRestTestChat(rig) + const running = rig.host.collaboratorsForTests().sessions.get(SESSION)!.child! + await rig.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'killed', + cause: 'unexpected-exit', + fence: running.fence, + acquisitionGeneration: running.generation! + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(rig.adapter.acquire).toHaveBeenCalledOnce() + + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + expect(rig.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + await rig.host.send(CALLER, restTestSend('again', fence)) + await vi.waitFor(() => expect(rig.adapter.acquire).toHaveBeenCalledTimes(2)) + }) + + it('whose settlement write failed is settled by the next send, which is delivered', async () => { + await foundRestTestChat(rig) + const open = rig.host.collaboratorsForTests().sessions.get(SESSION)! + const running = open.child! + // A turn in flight, so the exit has something to settle. + rig.adapter.acquire.mock.calls + .at(-1)?.[0] + .events?.appendItem( + { provider: 'codex', threadId: REST_TEST_THREAD, turnId: 'working', ordinal: 50 }, + { kind: 'turn', turnId: 'working', state: 'running' } + ) + await rig.host.flushStreamedEvents(SESSION) + vi.spyOn(open.journal, 'appendLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) + await rig.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + reason: 'killed', + cause: 'unexpected-exit', + fence: running.fence, + acquisitionGeneration: running.generation! + }) + await vi.waitFor(() => + expect(rig.store.getRecord(SESSION)?.lease.settlementRetryRequired).toBe(true) + ) + + const sent = await rig.host.send( + CALLER, + restTestSend('again', rig.store.getRecord(SESSION)!.lease.runtimeFence) + ) + + if (!sent.ok) { + throw new Error(`send refused: ${sent.refusal.code}`) + } + await rig.host.waitForSendSettlement(SESSION, sent.value.clientMessageId) + expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2) + expect(rig.store.getRecord(SESSION)?.lease.settlementRetryRequired).toBeUndefined() + }) +}) + +describe('every close withdraws what is queued (P2-29)', () => { + it.each([ + ['the close RPC', async () => void (await call('agentSession.close', { sessionId: SESSION }))], + ['the runtime chat close', async () => void (await closeStructuredAgentSessionChild(SESSION))], + [ + 'a discarded worker', + () => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the discard reads only this runtime member. + discardStructuredWorkerSession(SESSION, { + retireStructuredAgentSessionTabFromSnapshot: () => undefined + } as never) + ] + ])('%s rejects a message accepted before any start, and starts nothing', async (_, close) => { + await restingChat() + // The delivery loop has not reached its first start yet. + const { loop } = rig.host.collaboratorsForTests().conversationDelivery + vi.spyOn(loop, 'wake').mockImplementation(() => undefined) + const reader: unknown[] = [] + await rig.host.subscribe({ + id: 'reader', + sessionId: SESSION, + emit: (event) => reader.push(event) + }) + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + expect((await rig.host.send(CALLER, restTestSend('closed before it went', fence))).ok).toBe( + true + ) + + await close() + await vi.waitFor(() => + expect(JSON.stringify(reader)).toContain('provider_closed_before_delivery') + ) + expect(rig.host.hasSession(SESSION)).toBe(false) + expect(rig.adapter.acquire).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-background-task-capability.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-background-task-capability.test.ts index 2b62b00525e8..8a9e2267a5a8 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-background-task-capability.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-background-task-capability.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' import type { AgentSessionBackgroundTaskState } from '../../../../shared/agent-session-wire' import { AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, @@ -63,7 +63,6 @@ describe('background-task stop capability at the RPC boundary', () => { it.each(['snapshot', 'batch', 'reset'] as const)( 'gates the %s stream without changing the provider state', async (type) => { - hostCalls.hold = vi.fn(async () => undefined) hostCalls.subscribe.mockImplementation((input: AgentSessionSubscribeInput) => { const base = { sessionId: SESSION, fence: 1, backgroundTasks: TASKS } if (type === 'batch') { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-conversation-outline.ts b/src/main/runtime/rpc/methods/structured-agent-session-conversation-outline.ts index 99bf60da727d..08ecc212ec1b 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-conversation-outline.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-conversation-outline.ts @@ -7,7 +7,7 @@ import { readAgentSessionConversationOutline } from '../../../native-chat/agent-session-wire/agent-session-conversation-outline' import { defineMethod } from '../core' -import { requireStructuredHost as requireHost } from './structured-agent-session-gate' +import { requireInstalledStructuredHost } from './structured-agent-session-gate' import { OptionsParams } from './structured-agent-session-schemas' export const STRUCTURED_AGENT_SESSION_CONVERSATION_OUTLINE_METHODS = [ @@ -15,6 +15,8 @@ export const STRUCTURED_AGENT_SESSION_CONVERSATION_OUTLINE_METHODS = [ name: 'agentSession.conversationOutline', params: OptionsParams, handler: async (params, ctx) => - readAgentSessionConversationOutline(requireHost(ctx).journalSnapshot(params.sessionId)) + readAgentSessionConversationOutline( + await (await requireInstalledStructuredHost(ctx)).journalSnapshot(params.sessionId) + ) }) ] diff --git a/src/main/runtime/rpc/methods/structured-agent-session-gate-classification.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-gate-classification.test-fixture.ts index 72c52c888fd3..7b388f5eca22 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-gate-classification.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-gate-classification.test-fixture.ts @@ -21,15 +21,18 @@ export const CLEANUP_METHODS = [ params: { envelope: envelope(), turnId: 'turn-1' }, hostCall: 'cancel' }, + // A no-op kept for older clients: it answers without reaching the host. { method: 'agentSession.release', params: { sessionId: SESSION, holderId: 'surface-1' }, - hostCall: 'release' + hostCall: null, + result: { released: true } }, { method: 'agentSession.unsubscribe', params: { sessionId: SESSION }, - hostCall: 'unsubscribe' + hostCall: null, + result: { unsubscribed: true } } ] as const diff --git a/src/main/runtime/rpc/methods/structured-agent-session-gate.ts b/src/main/runtime/rpc/methods/structured-agent-session-gate.ts index 95e3e3d6eee7..f3ad78c5c5cc 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-gate.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-gate.ts @@ -73,10 +73,9 @@ export function requireStructuredCleanupHost(ctx: RpcContext): StructuredAgentSe return host } -/** Builds the host for the calls that address a session by durable record rather than by live - * state: attach, which is the only way a session comes into being, plus hold and reveal, which - * each reach for a record on disk this process may not have opened yet. Every other method - * addresses a session that must already be attached, and correctly reports absent when none is. */ +/** Builds the host for a call that may be the first this process sees. Every session is addressed + * by its durable record — a read opens a conversation at rest — so each call that reaches for one + * may meet a host nothing has built yet. */ export async function ensureStructuredHostInstalled(ctx: RpcContext): Promise { // Gated first: a client that cannot read structured sessions must not be able // to make the host exist, which is an observable side effect of the surface. @@ -89,6 +88,14 @@ export async function ensureStructuredHostInstalled(ctx: RpcContext): Promise { + await ensureStructuredHostInstalled(ctx) + return requireStructuredHost(ctx) +} + /** Mirrors the existing agent-session host-authority derivation so one client * gets one operation namespace across both surfaces. */ export function structuredCallerFor(ctx: RpcContext): StructuredAgentSessionCaller { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts index c3099d843ced..02099c9ddcf3 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts @@ -1,8 +1,8 @@ -// The wire half of a session's lifetime: who takes a hold, and what happens when they vanish. +// `agentSession.hold` / `release` are kept answering for clients that still send them, and do +// nothing else: a view never starts or keeps an agent. // -// Run against the REAL subscription registry rather than a stub, because the backstop being tested -// IS that registry's connection sweep — a stubbed `registerSubscriptionCleanup` would prove that -// the handler called a function, which is not the claim. +// Run against the REAL subscription registry rather than a stub, because the claim includes that a +// connection closing runs nothing hold-related — a stubbed registry could not show that. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -26,7 +26,6 @@ import { RpcDispatcher } from '../dispatcher' import { STRUCTURED_AGENT_SESSION_METHODS } from './structured-agent-session' const CONNECTION = 'connection-1' -const GRACE_MS = 5 const CLIENT = { clientId: 'device-1', clientKind: 'runtime' as const, @@ -90,7 +89,6 @@ beforeEach(async () => { journalRoot: root, claimKeyId: 'key-1', mintSpawnToken: () => 'spawn-a', - releaseGraceMs: GRACE_MS, now: () => NOW }) setStructuredAgentSessionHost(host) @@ -114,186 +112,85 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }) }) -describe('a client that holds a session', () => { - it('keeps the provider child while the hold stands', async () => { +describe('the hold surface, for clients that still call it', () => { + it('answers a hold without starting an agent or registering a cleanup', async () => { + await host.close(SESSION) + expect(host.hasSession(SESSION)).toBe(false) + const registered = vi.spyOn(runtime, 'registerOwnedSubscriptionCleanup') + const acquiresBefore = acquire.mock.calls.length + expect( await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) - ).toMatchObject({ ok: true }) + ).toMatchObject({ ok: true, result: { held: true } }) - await new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) - - expect(closeSession).not.toHaveBeenCalled() - expect(host.hasSession(SESSION)).toBe(true) + expect(acquire.mock.calls.length).toBe(acquiresBefore) + expect(registered).not.toHaveBeenCalled() + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') }) - it('releases it when the client says so', async () => { + it('answers a release without stopping anything, and a connection close runs nothing', async () => { await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) expect( await call('agentSession.release', { sessionId: SESSION, holderId: 'chat-1' }) - ).toMatchObject({ ok: true }) + ).toMatchObject({ ok: true, result: { released: true } }) + runtime.cleanupSubscriptionsForConnection(CONNECTION) + await new Promise((resolve) => setTimeout(resolve, 20)) - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledWith(SESSION) + expect(closeSession).not.toHaveBeenCalled() + expect(host.hasSession(SESSION)).toBe(true) }) - it('releases its hold and cleanup after the setting is disabled', async () => { - const release = vi.spyOn(host, 'release') - await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) + it('refuses a hold once the setting is off, and still answers a release', async () => { structuredNativeChatEnabled = false + expect( + await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) + ).toMatchObject({ ok: false }) expect( await call('agentSession.release', { sessionId: SESSION, holderId: 'chat-1' }) - ).toMatchObject({ ok: true }) - const releaseCallsAfterRpc = release.mock.calls.length - runtime.cleanupSubscriptionsForConnection(CONNECTION) - - expect(releaseCallsAfterRpc).toBe(2) - expect(release).toHaveBeenCalledTimes(releaseCallsAfterRpc) - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledWith(SESSION) - }) - - it('does not report success when no provider child can be acquired', async () => { - const response = await call('agentSession.hold', { - sessionId: 'session-missing', - holderId: 'chat-missing' - }) - - expect(response).toMatchObject({ - ok: false, - error: { code: 'agent_session_identity_required' } - }) - expect(host.isHeld('session-missing')).toBe(false) + ).toMatchObject({ ok: true, result: { released: true } }) + expect(closeSession).not.toHaveBeenCalled() }) -}) -describe('a client that disappears without cleanup', () => { - it('shares one child with a same-ID replacement that arrives while the first hold resumes', async () => { + it('answers a hold even when no agent could be started', async () => { await host.close(SESSION) - await host.restoreReadableSessions() - closeSession.mockClear() - acquire.mockClear() - const entered = Promise.withResolvers() - const gate = Promise.withResolvers() - const spawnChild = acquire.getMockImplementation()! - acquire.mockImplementationOnce(async (input) => { - entered.resolve() - await gate.promise - return spawnChild(input) - }) - try { - const params = { sessionId: SESSION, holderId: 'same-chat' } - const first = call('agentSession.hold', params) - await entered.promise - // Re-registering the cleanup id released the first hold; the replacement waits its turn - // behind the first hold's attach and finds the child it made. - const replacement = call('agentSession.hold', params) - gate.resolve() - - expect(await first).toMatchObject({ ok: true }) - expect(await replacement).toMatchObject({ ok: true }) - expect(acquire).toHaveBeenCalledOnce() - expect(host.isHeld(SESSION)).toBe(true) - await new Promise((resolve) => setTimeout(resolve, GRACE_MS * 4)) - expect(closeSession).not.toHaveBeenCalled() + acquire.mockRejectedValue(new Error('provider unavailable')) + const acquiresBefore = acquire.mock.calls.length - runtime.cleanupSubscriptionsForConnection(CONNECTION) - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledExactlyOnceWith(SESSION) - } finally { - gate.resolve() - } + expect( + await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) + ).toMatchObject({ ok: true, result: { held: true } }) + expect(acquire.mock.calls.length).toBe(acquiresBefore) }) +}) - it('lets a same-ID replacement make its own attempt when the first hold fails to acquire', async () => { +describe('a stream', () => { + it('reads a closed conversation without starting its agent', async () => { await host.close(SESSION) - await host.restoreReadableSessions() - closeSession.mockClear() - acquire.mockClear() - const entered = Promise.withResolvers() - const gate = Promise.withResolvers() - acquire.mockImplementationOnce(async () => { - entered.resolve() - await gate.promise - throw new Error('acquisition failed') - }) - try { - const params = { sessionId: SESSION, holderId: 'same-chat' } - const first = call('agentSession.hold', params) - await entered.promise - const replacement = call('agentSession.hold', params) - gate.resolve() - - expect(await first).toMatchObject({ ok: false }) - // One attempt per hold: the replacement's own succeeds, and the holder it re-took stands. - const replaced = await replacement - expect(replaced, JSON.stringify(replaced)).toMatchObject({ ok: true }) - expect(acquire).toHaveBeenCalledTimes(2) - expect(host.isHeld(SESSION)).toBe(true) - expect(closeSession).not.toHaveBeenCalled() - - runtime.cleanupSubscriptionsForConnection(CONNECTION) - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledExactlyOnceWith(SESSION) - } finally { - gate.resolve() - } - }) - - it('still releases the session when its transport closes', async () => { - await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) - - runtime.cleanupSubscriptionsForConnection(CONNECTION) - - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledWith(SESSION) - }) + expect(host.hasSession(SESSION)).toBe(false) + const acquiresBefore = acquire.mock.calls.length + const frames: unknown[] = [] - it('does not release a hold another connection is still holding', async () => { - await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) await dispatcher.dispatchStreaming( { - id: 'request-other', + id: 'request-subscribe', authToken: 'token', - method: 'agentSession.hold', - params: { sessionId: SESSION, holderId: 'chat-1' } + method: 'agentSession.subscribe', + params: { sessionId: SESSION } }, - () => {}, - { ...CLIENT, clientId: 'device-2', connectionId: 'connection-2' } + (raw) => frames.push(JSON.parse(raw)), + CLIENT ) - runtime.cleanupSubscriptionsForConnection(CONNECTION) - await new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) - - expect(closeSession).not.toHaveBeenCalled() - expect(host.hasSession(SESSION)).toBe(true) - }) - - it('does not let an old connection sweep release its same-document replacement', async () => { - await call('agentSession.hold', { sessionId: SESSION, holderId: 'chat-1' }) - await dispatcher.dispatchStreaming( - { - id: 'request-replacement', - authToken: 'token', - method: 'agentSession.hold', - params: { sessionId: SESSION, holderId: 'chat-1' } - }, - () => {}, - { ...CLIENT, connectionId: 'connection-2' } + expect(frames).toContainEqual( + expect.objectContaining({ result: expect.objectContaining({ type: 'snapshot' }) }) ) - - runtime.cleanupSubscriptionsForConnection(CONNECTION) - await new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) - - expect(closeSession).not.toHaveBeenCalled() - expect(host.hasSession(SESSION)).toBe(true) - - runtime.cleanupSubscriptionsForConnection('connection-2') - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) + expect(acquire.mock.calls.length).toBe(acquiresBefore) + expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') }) - it('releases a desktop subscription when its renderer transport dies', async () => { + it('keeps nothing alive when its transport dies', async () => { const transport = new AbortController() await dispatcher.dispatchStreaming( { @@ -310,56 +207,12 @@ describe('a client that disappears without cleanup', () => { clientCapabilities: CLIENT.clientCapabilities } ) - expect(host.isHeld(SESSION)).toBe(true) transport.abort() + await new Promise((resolve) => setTimeout(resolve, 20)) - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledWith(SESSION) - }) - - it('unsubscribes and releases stream retention after the setting is disabled', async () => { - await dispatcher.dispatchStreaming( - { - id: 'stream-disabled-cleanup', - authToken: 'token', - method: 'agentSession.subscribe', - params: { sessionId: SESSION } - }, - () => {}, - CLIENT - ) - expect(host.isHeld(SESSION)).toBe(true) - structuredNativeChatEnabled = false - - expect( - await call('agentSession.unsubscribe', { - sessionId: SESSION, - subscriptionId: 'stream-disabled-cleanup' - }) - ).toMatchObject({ ok: true }) - - await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) - expect(closeSession).toHaveBeenCalledWith(SESSION) - }) - - it('does not let a stream alone resume a released session', async () => { - await host.close(SESSION) - expect(host.hasSession(SESSION)).toBe(false) - await host.restoreReadableSessions() - expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - - await dispatcher.dispatchStreaming( - { - id: 'request-subscribe', - authToken: 'token', - method: 'agentSession.subscribe', - params: { sessionId: SESSION } - }, - () => {}, - CLIENT - ) - - expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + // The idle sweep, not a departing reader, is what stops an agent. + expect(closeSession).not.toHaveBeenCalled() + expect(host.hasSession(SESSION)).toBe(true) }) }) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-hold.ts b/src/main/runtime/rpc/methods/structured-agent-session-hold.ts index 54753fcb8894..e1975f1c7255 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-hold.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-hold.ts @@ -1,64 +1,31 @@ -// `agentSession.hold` / `agentSession.release` — a surface saying it is bound to a session. +// `agentSession.hold` / `agentSession.release` — kept answering for clients that still send them. // -// The holder identity is scoped to the CONNECTION, not taken from the client verbatim: two clients -// are free to name their surfaces the same thing, and a hold that collides is one that a stranger -// can release. -// -// The registered cleanup is the backstop, and the ONLY thing that covers a client which vanishes — -// a paired client that disconnects mid-turn never gets to send its release. Registering it before taking -// the hold is deliberate: re-registering an id runs the previous cleanup synchronously, so the -// stale release lands before this hold rather than after it. +// A view no longer decides whether an agent runs: a send starts one, and the idle sweep stops it. +// So both are no-ops. `hold` still builds the host, because shipped mobile builds build it through +// `hold` before they subscribe. Delete both, and their allowlist entries, once +// MIN_COMPATIBLE_RUNTIME_CLIENT_VERSION passes every client that still calls them. -import { defineMethod, type RpcContext } from '../core' +import { defineMethod } from '../core' import { - ensureStructuredHostInstalled, - requireStructuredCleanupHost, - requireStructuredHost + requireInstalledStructuredHost, + requireStructuredCleanupHost } from './structured-agent-session-gate' import { HoldParams } from './structured-agent-session-schemas' -const HOLD_CLEANUP_PREFIX = 'agentSession.hold' - -function holderKeyFor(ctx: RpcContext, holderId: string): string { - const client = ctx.clientId?.trim() || (ctx.clientKind ?? 'runtime') - return `${ctx.connectionId ?? 'local'}:${client}:${holderId}` -} - -function holdCleanupIdFor(sessionId: string, holderKey: string): string { - return `${HOLD_CLEANUP_PREFIX}:${holderKey}:${sessionId}` -} - export const STRUCTURED_AGENT_SESSION_HOLD_METHODS = [ defineMethod({ name: 'agentSession.hold', params: HoldParams, - handler: async (params, ctx) => { - await ensureStructuredHostInstalled(ctx) - const host = requireStructuredHost(ctx) - const holderKey = holderKeyFor(ctx, params.holderId) - const registration = ctx.runtime.registerOwnedSubscriptionCleanup( - holdCleanupIdFor(params.sessionId, holderKey), - () => host.release(params.sessionId, holderKey), - ctx.connectionId - ) - try { - await host.hold(params.sessionId, holderKey) - } catch (error) { - registration.releaseIfCurrent() - throw error - } + handler: async (_params, ctx) => { + await requireInstalledStructuredHost(ctx) return { held: true as const } } }), defineMethod({ name: 'agentSession.release', params: HoldParams, - handler: async (params, ctx) => { - const host = requireStructuredCleanupHost(ctx) - const holderKey = holderKeyFor(ctx, params.holderId) - host.release(params.sessionId, holderKey) - // Retires the backstop too; its release is a no-op against a holder already gone. - ctx.runtime.cleanupSubscription(holdCleanupIdFor(params.sessionId, holderKey)) + handler: async (_params, ctx) => { + requireStructuredCleanupHost(ctx) return { released: true as const } } }) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-options-read.ts b/src/main/runtime/rpc/methods/structured-agent-session-options-read.ts index b74b9cacfa66..19e936895771 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-options-read.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-options-read.ts @@ -1,22 +1,26 @@ // The options read surface: what a session reports about itself, and the // host's stored model catalog behind the picker. // -// `agentSession.options` serializes on the session because it asks the live -// provider what is selected. `agentSession.modelCatalog` deliberately does +// `agentSession.options` answers at rest from the record and the catalog, and +// asks the live provider only when one runs. `agentSession.modelCatalog` deliberately does // neither — answering from the host store is what lets a picker render while // an attach is still running. It is additive: an older host answers // `method_not_found` (or `forbidden` through the mobile allowlist gate), and // the client keeps its static seed. import { defineMethod } from '../core' -import { requireStructuredHost as requireHost } from './structured-agent-session-gate' +import { + requireInstalledStructuredHost, + requireStructuredHost as requireHost +} from './structured-agent-session-gate' import { ModelCatalogParams, OptionsParams } from './structured-agent-session-schemas' export const STRUCTURED_AGENT_SESSION_OPTIONS_READ_METHODS = [ defineMethod({ name: 'agentSession.options', params: OptionsParams, - handler: async (params, ctx) => requireHost(ctx).readOptions(params.sessionId) + handler: async (params, ctx) => + (await requireInstalledStructuredHost(ctx)).readOptions(params.sessionId) }), defineMethod({ name: 'agentSession.modelCatalog', diff --git a/src/main/runtime/rpc/methods/structured-agent-session-restart-resume.ts b/src/main/runtime/rpc/methods/structured-agent-session-restart-resume.ts index b3a70334f409..a48f897a005e 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-restart-resume.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-restart-resume.ts @@ -64,20 +64,15 @@ export const STRUCTURED_AGENT_SESSION_RESTART_RESUME_METHODS = [ } }), defineMethod({ - // Reattach only, no send. No Orca surface calls it now — the desktop prompt's single action is - // resume-and-continue — but it is a PUBLISHED wire method, so dropping it is a wire removal an - // older or non-desktop client would meet as an unknown method. + // Reattach only, no send. Reattaching is nothing now — an agent starts only for work — so this + // answers that nothing was resumed. No Orca surface calls it, but it is a PUBLISHED wire + // method, so dropping it is a wire removal an older client would meet as an unknown method. name: 'agentSession.restartResume', params: RestartResumeParams, - handler: async (params, ctx) => { + handler: async (_params, ctx) => { await ensureStructuredHostInstalled(ctx) - const host = requireStructuredHost(ctx) - return { - results: await host.restartResume.resume( - params.sessionIds, - structuredCallerFor(ctx).callerKey - ) - } + requireStructuredHost(ctx) + return { results: [] } } }) ] diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index a9cf5cd79576..9a0f3902496b 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -203,8 +203,7 @@ export function hostStub(): StructuredAgentSessionHost { subscribeStatus: vi.fn((subscriber: StructuredAgentSessionStatusSubscriber) => statusFeed().subscribe(subscriber) ), - unsubscribe: vi.fn(), - release: vi.fn() + unsubscribe: vi.fn() }) return hostCalls as unknown as StructuredAgentSessionHost } diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts new file mode 100644 index 000000000000..9ae846df6fc0 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts @@ -0,0 +1,84 @@ +// Which clients get a send answered at acceptance, and which have their reply held until the +// message is handed over: a client that cannot show a rejection after `pending` must not see one. + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' +import { DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES } from '../../../ipc/desktop-renderer-runtime-capabilities' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' +import { + call, + clearStructuredHostStub, + hostCalls, + installStructuredHostStub, + SESSION, + sendParams, + STRUCTURED_CLIENT +} from './structured-agent-session-rpc.test-fixture' + +beforeEach(() => { + installStructuredHostStub() +}) + +afterEach(() => { + clearStructuredHostStub() +}) + +describe('agentSession.send reply timing', () => { + it('holds a pending reply until handover for a client that cannot show a later rejection', async () => { + hostCalls.send.mockResolvedValueOnce(pendingSendResult()) + hostCalls.waitForSendSettlement.mockResolvedValueOnce(undefined) + + await call('agentSession.send', sendParams(), STRUCTURED_CLIENT) + + expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', { + until: 'handed-over', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS + }) + }) + + it('answers at acceptance for the local desktop and paired desktop clients (W2)', async () => { + for (const clientCapabilities of [ + DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES, + // A paired desktop gains the structured surface as its own capability; the reply rule rides + // on the list it already sends. + [...ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + ]) { + hostCalls.send.mockResolvedValueOnce(pendingSendResult()) + const response = await call('agentSession.send', sendParams(), { + clientKind: 'runtime', + clientCapabilities: [...clientCapabilities] + }) + expect(response).toMatchObject({ + ok: true, + result: { value: { submission: { dispatchState: 'pending' } } } + }) + } + expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled() + }) +}) + +function pendingSendResult() { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-a', sequence: 1 }, + value: { + clientMessageId: 'client-1', + submission: { + clientMessageId: 'client-1', + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending' as const, + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true as const + } + } + } +} diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts index 8b9488699904..5fde5e0f6e49 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts @@ -1,26 +1,43 @@ -import { AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, + AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import type { RpcContext } from '../core' import { requireStructuredHost, structuredCallerFor } from './structured-agent-session-gate' +/** + * A send answers once the host accepts it. A client that predates that answer cannot show a + * message rejected after it, so its reply is held until the message is handed over or rejected; + * one that predates pending replies at all waits, as before, for the provider's answer. + */ export async function sendStructuredAgentSessionForClient( params: Parameters[1], context: RpcContext ) { const host = requireStructuredHost(context) const result = await host.send(structuredCallerFor(context), params) + const capabilities = context.clientCapabilities ?? [] if ( !result.ok || result.value.submission.dispatchState !== 'pending' || context.clientKind === undefined || - context.clientCapabilities?.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + capabilities.includes(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) ) { return result } + // The start that used to run before the reply now runs after acceptance, so both waits cover it. const settled = await host.waitForSendSettlement( params.envelope.sessionId, result.value.clientMessageId, - context.signal + { + until: capabilities.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + ? 'handed-over' + : 'answered', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS, + ...(context.signal ? { signal: context.signal } : {}) + } ) return settled ? { ...result, ...settled } : result } diff --git a/src/main/runtime/rpc/methods/structured-agent-session-turn-item-capability.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-turn-item-capability.test.ts index 0393cf216c84..473f63313729 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-turn-item-capability.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-turn-item-capability.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' import type { AgentSessionHistoryPage, @@ -87,7 +87,6 @@ describe('turn item capability at the RPC boundary', () => { it.each(['snapshot', 'batch', 'reset'] as const)( 'downgrades the %s stream only for a legacy reader', async (type) => { - hostCalls.hold = vi.fn(async () => undefined) hostCalls.subscribe.mockImplementation((input: AgentSessionSubscribeInput) => { const base = { sessionId: SESSION, fence: 1 } if (type === 'batch') { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-unsubscribe.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-unsubscribe.test.ts new file mode 100644 index 000000000000..a31a8bc8d186 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-unsubscribe.test.ts @@ -0,0 +1,91 @@ +// A transcript stream ends when its client says so: the host drops that subscriber, so nothing is +// derived or sent for it any more, and a sibling stream of the same session keeps going. + +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' +import { + createRestTestRig, + foundRestTestChat, + REST_TEST_CALLER as CALLER, + REST_TEST_SESSION as SESSION, + restTestSend, + type RestTestRig +} from '../../../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' +import { OrcaRuntimeService } from '../../orca-runtime' +import type { RpcResponse } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { STRUCTURED_AGENT_SESSION_METHODS } from './structured-agent-session' + +const CLIENT = { + clientId: 'device-1', + clientKind: 'runtime' as const, + clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY], + connectionId: 'connection-1' +} + +let rig: RestTestRig +let dispatcher: RpcDispatcher + +function subscriberCount(): number { + return rig.host.collaboratorsForTests().subscribers.subscriberCountForTests(SESSION) +} + +async function stream(id: string, frames: RpcResponse[]): Promise { + await dispatcher.dispatchStreaming( + { id, authToken: 'token', method: 'agentSession.subscribe', params: { sessionId: SESSION } }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the dispatcher writes RpcResponse JSON. + (raw) => frames.push(JSON.parse(raw) as RpcResponse), + CLIENT + ) +} + +beforeEach(async () => { + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) + setStructuredAgentSessionHost(rig.host) + const runtime = new OrcaRuntimeService() + vi.spyOn(runtime, 'getClientSettings').mockImplementation( + () => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the RPC gate reads only this one setting. + ({ experimentalStructuredNativeChat: true }) as ReturnType< + OrcaRuntimeService['getClientSettings'] + > + ) + dispatcher = new RpcDispatcher({ runtime, methods: STRUCTURED_AGENT_SESSION_METHODS }) +}) + +afterEach(async () => { + setStructuredAgentSessionHost(null) + await rig.dispose() +}) + +it('drops the subscriber a client unsubscribes, and keeps its sibling (U-02)', async () => { + await foundRestTestChat(rig) + const ended: RpcResponse[] = [] + const kept: RpcResponse[] = [] + await stream('frame-a', ended) + await stream('frame-b', kept) + expect(subscriberCount()).toBe(2) + + const replies: RpcResponse[] = [] + await dispatcher.dispatchStreaming( + { + id: 'frame-c', + authToken: 'token', + method: 'agentSession.unsubscribe', + params: { sessionId: SESSION, subscriptionId: 'frame-a' } + }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the dispatcher writes RpcResponse JSON. + (raw) => replies.push(JSON.parse(raw) as RpcResponse), + CLIENT + ) + expect(replies[0]).toMatchObject({ ok: true }) + expect(subscriberCount()).toBe(1) + + const before = ended.length + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + await rig.host.send(CALLER, restTestSend('after the unsubscribe', fence)) + await vi.waitFor(() => expect(kept.length).toBeGreaterThan(1)) + // The ended stream's own `end` frame is all it gets. + expect(ended.slice(before).every((frame) => JSON.stringify(frame).includes('"end"'))).toBe(true) +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session.test.ts b/src/main/runtime/rpc/methods/structured-agent-session.test.ts index 38b55fd70ebd..a8378489c029 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, RUNTIME_CAPABILITIES, RUNTIME_PROTOCOL_VERSION, @@ -12,6 +13,7 @@ import { STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import { computeAgentSessionPayloadFingerprint } from '../../../../shared/agent-session-mutation-envelope' import { ALL_RPC_METHODS } from './index' import { STRUCTURED_AGENT_SESSION_METHODS } from './structured-agent-session' @@ -149,6 +151,8 @@ describe('capability gating', () => { it('advertises the capability without bumping the protocol version', () => { expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + // A client tells a host that accepts first, and admits a writer-free Stop before a turn, by it. + expect(RUNTIME_CAPABILITIES).toContain(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY) // Separate from the structured capability on purpose: a host can serve the rest of the @@ -252,11 +256,11 @@ describe('capability gating', () => { signal: controller.signal }) - expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith( - SESSION, - 'client-1', - controller.signal - ) + expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', { + until: 'answered', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS, + signal: controller.signal + }) expect(response).toMatchObject({ ok: true, result: { @@ -302,36 +306,6 @@ describe('capability gating', () => { }) }) - it('returns durable pending immediately to clients that understand admission', async () => { - hostCalls.send.mockResolvedValueOnce({ - ok: true, - replayed: false, - fence: 1, - cursor: { epoch: 'epoch-a', sequence: 1 }, - value: { - clientMessageId: 'client-1', - submission: { - clientMessageId: 'client-1', - fence: 1, - payloadFingerprint: 'fingerprint', - dispatchState: 'pending', - providerItemId: null, - reason: null, - submittedAt: 1, - resolvedAt: null - } - } - }) - - const response = await call('agentSession.send', sendParams(), STRUCTURED_CLIENT) - - expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled() - expect(response).toMatchObject({ - ok: true, - result: { value: { submission: { dispatchState: 'pending' } } } - }) - }) - it('requires the host structured-chat setting for mobile clients', async () => { const response = await call('agentSession.send', sendParams(), STRUCTURED_MOBILE_CLIENT, { getClientSettings: () => ({ experimentalStructuredNativeChat: false }) @@ -363,7 +337,9 @@ describe('capability gating', () => { ok: false, error: { message: expect.stringContaining('structured_agent_session_unsupported') } }) - expect(hostCalls[hostCall]).not.toHaveBeenCalled() + if (hostCall !== null) { + expect(hostCalls[hostCall]).not.toHaveBeenCalled() + } } ) diff --git a/src/main/runtime/rpc/methods/structured-agent-session.ts b/src/main/runtime/rpc/methods/structured-agent-session.ts index ebff420cb002..a238c3050e4a 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.ts @@ -18,6 +18,7 @@ import { import { defineMethod, defineStreamingMethod, type RpcContext } from '../core' import { ensureStructuredHostInstalled as ensureHostInstalled, + requireInstalledStructuredHost as requireInstalledHost, requireStructuredCapability, requireStructuredCleanupHost, requireStructuredHost as requireHost, @@ -235,19 +236,20 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ defineMethod({ name: 'agentSession.handoffStatus', params: HandoffStatusParams, - handler: async (params, ctx) => requireHost(ctx).handoffStatus(params.sessionId) + handler: async (params, ctx) => + (await requireInstalledHost(ctx)).handoffStatus(params.sessionId) }), defineMethod({ name: 'agentSession.commands', params: OptionsParams, - handler: async (params, ctx) => requireHost(ctx).readCommands(params.sessionId) + handler: async (params, ctx) => (await requireInstalledHost(ctx)).readCommands(params.sessionId) }), defineMethod({ name: 'agentSession.history', params: HistoryParams, handler: async (params, ctx) => projectTurnItemHistory( - projectBackgroundTaskHistory(requireHost(ctx).history(params), ctx), + projectBackgroundTaskHistory(await (await requireInstalledHost(ctx)).history(params), ctx), ctx ) }), @@ -255,25 +257,17 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ name: 'agentSession.subscribe', params: SubscribeParams, handler: async (params, ctx, emit) => { - const host = requireHost(ctx) + const host = await requireInstalledHost(ctx) const subscriptionId = subscriptionIdFor(ctx, params.sessionId) - // A live stream is a surface too: it keeps a session from being evicted while it is read and - // releases that retention when the transport dies without a word. - // - // Retain-only: reading history must never be what starts a provider process. Current clients - // explicitly hold every open surface before subscribing. - const streamHolder = `subscription:${subscriptionId}` + // A stream reads; it never keeps an agent alive or starts one. let dispose = (): void => {} - const stream = bindStructuredAgentSessionStream(ctx, subscriptionId, () => { - dispose() - host.release(params.sessionId, streamHolder) - }) + const stream = bindStructuredAgentSessionStream(ctx, subscriptionId, () => dispose()) if (stream.isClosed()) { return } - // The host emits the opening snapshot (or the missed batch) synchronously - // inside open(), so nothing between here and there can interleave. - dispose = host.subscribe({ + // Resolves once the conversation is open and the opening snapshot (or the missed batch) is + // emitted; a close that raced the open disposes what it bound. + dispose = await host.subscribe({ id: subscriptionId, sessionId: params.sessionId, emit: (event) => emit(projectTurnItemEvent(projectBackgroundTaskEvent(event, ctx), ctx)), @@ -281,14 +275,6 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ }) if (stream.isClosed()) { dispose() - } else { - // Fire-and-forget, but never unhandled: a resume that refuses leaves the stream holding a - // readable session, which is exactly what the client sees anyway. - void host - .hold(params.sessionId, streamHolder, { resume: false }) - .catch((error: unknown) => - console.warn('[agent-session] stream hold failed', params.sessionId, error) - ) } } }), diff --git a/src/main/runtime/rpc/methods/structured-chat-tab-table.test.ts b/src/main/runtime/rpc/methods/structured-chat-tab-table.test.ts index f76b3ae6cb74..7cf91e754513 100644 --- a/src/main/runtime/rpc/methods/structured-chat-tab-table.test.ts +++ b/src/main/runtime/rpc/methods/structured-chat-tab-table.test.ts @@ -277,10 +277,7 @@ describe('a chat tab across /clear', () => { await openHost() expect(store.getSessionTabId(replacement)).toBe(SOURCE_TAB) - expect(host.getPersistedVisibleSessionTabIndex()).toEqual({ - present: true, - sessionIds: [replacement] - }) + expect(host.listVisibleSessionIds()).toEqual([replacement]) }) it('gives a reopened cleared conversation the same id when an older build drops the table', async () => { @@ -387,7 +384,7 @@ describe('a create that reserves its tab', () => { await host.flushAllStreamedEvents() await openHost() - expect(host.getPersistedVisibleSessionTabIndex().sessionIds).toEqual([]) + expect(host.listVisibleSessionIds()).toEqual([]) expect(await createChat('session-bravo', 'reserved-tab')).toMatchObject({ ok: true, value: { tabId: 'reserved-tab' } diff --git a/src/main/runtime/rpc/methods/structured-worker-read-cursor.test.ts b/src/main/runtime/rpc/methods/structured-worker-read-cursor.test.ts index 1a97223bc395..b376f2c80d9e 100644 --- a/src/main/runtime/rpc/methods/structured-worker-read-cursor.test.ts +++ b/src/main/runtime/rpc/methods/structured-worker-read-cursor.test.ts @@ -79,7 +79,7 @@ function read(cursor?: string, limit?: number) { }) } -function textsOf(result: ReturnType): string[] { +function textsOf(result: Awaited>): string[] { return result.transcript.messages.map((entry) => entry.blocks.map((block) => ('text' in block ? block.text : '')).join('') ) @@ -90,57 +90,57 @@ describe('the structured worker-read cursor over a mutating journal', () => { hostRef.current = null }) - it('refuses to resume when an already-delivered item was revised in place', () => { + it('refuses to resume when an already-delivered item was revised in place', async () => { // The `"hel"` / `"hello"` defect. The caller is handed a coalesced snapshot, resumes past it, // and the item is later revised at its original sequence — under the old anchor the resume was // accepted and that revision was never delivered to anyone. installJournal([message('i1', 'hel'), message('i2', 'second')]) - const first = read(undefined, 1) + const first = await read(undefined, 1) expect(textsOf(first)).toEqual(['hel']) installJournal([message('i1', 'hello world', 2), message('i2', 'second')]) - expect(() => read(first.cursor)).toThrow(/source changed/i) + await expect(read(first.cursor)).rejects.toThrow(/source changed/i) }) - it('refuses to resume when a resolved prompt inserts ahead of the caller position', () => { + it('refuses to resume when a resolved prompt inserts ahead of the caller position', async () => { // Duplication. A pending approval projects to null, so resolving it inserts a message in the // MIDDLE; the oldest item never moved, so the old anchor accepted a now-stale index and the // caller re-read content it already had. installJournal([message('i1', 'first'), approval('i2', false), message('i3', 'second')]) - const first = read(undefined, 2) + const first = await read(undefined, 2) expect(textsOf(first)).toEqual(['first', 'second']) installJournal([message('i1', 'first'), approval('i2', true, 2), message('i3', 'second')]) - expect(() => read(first.cursor)).toThrow(/source changed/i) + await expect(read(first.cursor)).rejects.toThrow(/source changed/i) }) - it('still resumes across a page boundary when only unread tail items change', () => { + it('still resumes across a page boundary when only unread tail items change', async () => { // The reason this is prefix-scoped and not whole-page: during an active turn the coalescer // revises the streaming item every 60ms. Fingerprinting the whole page would invalidate the // cursor continuously — a useless verb — while the worker is working. installJournal([message('i1', 'first'), message('i2', 'streaming')]) - const first = read(undefined, 1) + const first = await read(undefined, 1) expect(textsOf(first)).toEqual(['first']) installJournal([message('i1', 'first'), message('i2', 'streaming more', 7)]) - const second = read(first.cursor) + const second = await read(first.cursor) expect(textsOf(second)).toEqual(['streaming more']) }) - it('delivers every message exactly once when nothing below the cursor changes', () => { + it('delivers every message exactly once when nothing below the cursor changes', async () => { // The property the two refusals above protect: no omission, no duplication. installJournal([message('i1', 'a'), message('i2', 'b'), message('i3', 'c')]) - const first = read(undefined, 2) - const second = read(first.cursor, 2) + const first = await read(undefined, 2) + const second = await read(first.cursor, 2) expect([...textsOf(first), ...textsOf(second)]).toEqual(['a', 'b', 'c']) }) - it('still refuses when the window slides off the front', () => { + it('still refuses when the window slides off the front', async () => { // The case the old anchor DID catch, and which the prefix scoping must not lose: a slide // shifts every index. installJournal([message('i1', 'a'), message('i2', 'b')]) - const first = read(undefined, 1) + const first = await read(undefined, 1) installJournal([message('i2', 'b'), message('i3', 'c')]) - expect(() => read(first.cursor)).toThrow(/source changed/i) + await expect(read(first.cursor)).rejects.toThrow(/source changed/i) }) }) diff --git a/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts b/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts index b48dccf52a39..a8b8d4a07c93 100644 --- a/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts +++ b/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts @@ -43,10 +43,12 @@ function installHost(options: { closeThrows?: boolean; lease?: Record attached, + reconcileRestartLeases: async () => {}, hold: async () => {}, release: () => {}, subscribe: () => () => {}, diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-mobile-method-allowlist.ts b/src/main/runtime/runtime-rpc/runtime-rpc-mobile-method-allowlist.ts index ce67fb94aff7..a543a8da45d0 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-mobile-method-allowlist.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-mobile-method-allowlist.ts @@ -230,6 +230,8 @@ export const MOBILE_RPC_METHOD_ALLOWLIST = new Set([ 'agentSession.history', 'agentSession.subscribe', 'agentSession.unsubscribe', + // No-ops on a current host; kept until MIN_COMPATIBLE_RUNTIME_CLIENT_VERSION passes the + // mobile builds that still call them. 'agentSession.hold', 'agentSession.release', 'nativeChat.readSession', diff --git a/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts b/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts index b1e43a9025d2..27d5ba48b299 100644 --- a/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts +++ b/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts @@ -89,7 +89,7 @@ async function awaitingApproval() { SESSION, { journal, - hasProviderChild: true, + child: { phase: 'ready' as const }, params: { location: { executionHostId: 'local' as const, diff --git a/src/main/runtime/saved-structured-agent-session-restoration.ts b/src/main/runtime/saved-structured-agent-session-restoration.ts index 71d617de81a0..a784eea58e40 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.ts @@ -1,6 +1,11 @@ import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { Tab } from '../../shared/tab-types' import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' +import { + getActiveSidebarWorkspaceId, + normalizeWorkspaceSessionKeyToWorkspaceId +} from '../../shared/workspace-scope' +import type { StructuredAgentSessionStartupPriority } from '../native-chat/agent-session-wire/structured-agent-session-restart-restore' function savedSessionId(tab: Tab): string | null { if (tab.executionHostId && tab.executionHostId !== LOCAL_EXECUTION_HOST_ID) { @@ -41,3 +46,24 @@ export function collectSavedStructuredAgentSessionIds( } return selected } + +/** Which workspaces the startup pass reaches first: the one the window last showed, then any with + * a saved tab. Null without a saved desktop session (`orca serve`, SSH), keeping listing order. */ +export function structuredAgentSessionStartupPriority( + session: WorkspaceSessionState | null +): StructuredAgentSessionStartupPriority | undefined { + if (!session) { + return undefined + } + const withTabs = [ + ...Object.keys(session.unifiedTabs ?? {}), + ...Object.keys(session.tabsByWorktree ?? {}) + ].map(normalizeWorkspaceSessionKeyToWorkspaceId) + return { + activeWorkspaceId: getActiveSidebarWorkspaceId( + session.activeWorkspaceKey ?? null, + session.activeWorktreeId + ), + visibleWorkspaceIds: new Set(withTabs) + } +} diff --git a/src/main/runtime/structured-agent-session-close.ts b/src/main/runtime/structured-agent-session-close.ts index 9afb2723a882..668b7f218903 100644 --- a/src/main/runtime/structured-agent-session-close.ts +++ b/src/main/runtime/structured-agent-session-close.ts @@ -3,8 +3,8 @@ * * Extracted from `stopStructuredWorker` so that orchestration settlement and worktree teardown * close a session the SAME way rather than one of them inventing a shorter version. Everything - * dispatch-shaped — dropping the hold, the redrive subscription and the parked mail — stays with - * the caller that has a dispatch; this is only the child. + * dispatch-shaped — dropping the redrive subscription, the registry entry and the parked mail — + * stays with the caller that has a dispatch; this is only the child. * * `host.close` returns void and keeps a failed close indexed for retry, so the only settlement * evidence is the observation AFTER it: a session the host no longer holds and whose lease is no @@ -15,7 +15,10 @@ import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wi import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import type { OrcaRuntimeService } from './orca-runtime' import { retireSettledStructuredWorkerTab } from './structured-agent-session-tab-retirement' -import { observeStructuredWorker } from './structured-worker-authority' +import { + observeStructuredWorker, + structuredSessionCloseSettled +} from './structured-worker-authority' export type StructuredAgentSessionCloseOutcome = { stopped: boolean @@ -32,8 +35,8 @@ export type StructuredAgentSessionCloseOptions = { /** * Runs after the close is issued and BEFORE the proof is read. * - * Not after: an unsettled close returns early, so a dispatch that released its hold there would - * keep the child un-evictable for the life of the app. Every settlement has to reach it. + * Not after: an unsettled close returns early, so a dispatch released there would keep its + * redrive subscription nudging a session no dispatch owns. Every settlement has to reach it. */ afterClose?: () => void /** @@ -86,8 +89,8 @@ export async function closeStructuredAgentSessionChild( } } options.afterClose?.() - const observation = observeStructuredWorker({ sessionId }) - if (observation.status !== 'exited') { + if (!structuredSessionCloseSettled(sessionId)) { + const observation = observeStructuredWorker({ sessionId }) await restorePersistedTabVisibility(host, sessionId, restoreTabIfCloseFails) return { stopped: false, @@ -137,7 +140,7 @@ async function restorePersistedTabVisibility( sessionId: string, tabId: string | null ): Promise { - if (tabId === null || observeStructuredWorker({ sessionId }).status === 'exited') { + if (tabId === null || structuredSessionCloseSettled(sessionId)) { return } try { diff --git a/src/main/runtime/structured-agent-session-integration-replay.test.ts b/src/main/runtime/structured-agent-session-integration-replay.test.ts index 4fa390be0fb9..a4e2500ffc46 100644 --- a/src/main/runtime/structured-agent-session-integration-replay.test.ts +++ b/src/main/runtime/structured-agent-session-integration-replay.test.ts @@ -317,11 +317,14 @@ describe('a structured codex session over agentSession.*', () => { body } + const turnStarts = () => codex.live().calls.filter((entry) => entry.method === 'turn/start') await ok('agentSession.send', params) + // Accepted first; the delivery loop hands it over once. + await vi.waitFor(() => expect(turnStarts()).toHaveLength(1)) const replay = await call('agentSession.send', params) expect(replay).toMatchObject({ ok: true, result: { ok: true, replayed: true } }) - expect(codex.live().calls.filter((entry) => entry.method === 'turn/start')).toHaveLength(1) + expect(turnStarts()).toHaveLength(1) }) it('joins an acquired attach through journal bind before draining final rows', async () => { diff --git a/src/main/runtime/structured-agent-session-integration.test.ts b/src/main/runtime/structured-agent-session-integration.test.ts index 3f8adae4906a..1b7136a996df 100644 --- a/src/main/runtime/structured-agent-session-integration.test.ts +++ b/src/main/runtime/structured-agent-session-integration.test.ts @@ -218,6 +218,11 @@ function createIntentParams() { } let codex: CodexScript +/** Accepted first; the delivery loop hands the send over as `turn/start` after the reply. */ +const handedOverAs = (params: Record) => + vi.waitFor(() => + expect(codex.live().calls.at(-1)).toMatchObject({ method: 'turn/start', params }) + ) let root: string let dispatcher: RpcDispatcher let bootEnvironmentReads: number @@ -284,10 +289,10 @@ function textOf(item: AgentJournalRenderItem): string { } /** The durable submission row, which settlement rewrites after the send returns. */ -function submissionOf(clientMessageId: string): AgentJournalSubmission | undefined { - return getStructuredAgentSessionHost() - ?.journalSnapshot(SESSION) - .submissions.find((entry) => entry.clientMessageId === clientMessageId) +async function submissionOf(clientMessageId: string): Promise { + return (await getStructuredAgentSessionHost()?.journalSnapshot(SESSION))?.submissions.find( + (entry) => entry.clientMessageId === clientMessageId + ) } async function historyPage( @@ -466,10 +471,7 @@ describe('a structured codex session over agentSession.*', () => { // send coalesced into a running turn is answered with that turn's id, so // which message landed where is knowable only from the echo. expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null }) - expect(codex.live().calls.at(-1)).toMatchObject({ - method: 'turn/start', - params: { threadId: THREAD, clientUserMessageId: sent.clientMessageId } - }) + await handedOverAs({ threadId: THREAD, clientUserMessageId: sent.clientMessageId }) codex.notify('turn/started', { turn: { id: TURN } }) // Codex echoes the message back carrying the `clientId` it was sent under, @@ -493,8 +495,8 @@ describe('a structured codex session over agentSession.*', () => { // The echo is the first item of this turn, so the settled key is ordinal 0 — // minted by the same `identityFor` a history replay computes with, rather // than guessed from the turn/start response. - await vi.waitFor(() => - expect(submissionOf(sent.clientMessageId)).toMatchObject({ + await vi.waitFor(async () => + expect(await submissionOf(sent.clientMessageId)).toMatchObject({ dispatchState: 'accepted', providerItemId: `codex:${THREAD}:${TURN}:0` }) @@ -557,14 +559,11 @@ describe('a structured codex session over agentSession.*', () => { // "delivery unconfirmed" — it carries no identity yet, because the response // to a coalesced send names the running turn rather than this message. expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null }) - expect(codex.live().calls.at(-1)).toMatchObject({ - method: 'turn/start', - params: { - threadId: THREAD, - clientUserMessageId: sent.clientMessageId, - model: 'gpt-live', - effort: 'high' - } + await handedOverAs({ + threadId: THREAD, + clientUserMessageId: sent.clientMessageId, + model: 'gpt-live', + effort: 'high' }) // ── stream ────────────────────────────────────────────────────────────── @@ -585,8 +584,8 @@ describe('a structured codex session over agentSession.*', () => { expect(itemsOf(stream).filter((item) => textOf(item) === 'list files')).toHaveLength(1) // Settled from the echo's own journal identity, so it is by construction the // key a replay recomputes for this row. - await vi.waitFor(() => - expect(submissionOf(sent.clientMessageId)).toMatchObject({ + await vi.waitFor(async () => + expect(await submissionOf(sent.clientMessageId)).toMatchObject({ dispatchState: 'accepted', providerItemId: `codex:${THREAD}:${TURN}:0` }) diff --git a/src/main/runtime/structured-agent-session-rollback-compatibility.test.ts b/src/main/runtime/structured-agent-session-rollback-compatibility.test.ts index 293ec194aae7..733a7cf42b55 100644 --- a/src/main/runtime/structured-agent-session-rollback-compatibility.test.ts +++ b/src/main/runtime/structured-agent-session-rollback-compatibility.test.ts @@ -99,17 +99,22 @@ describe('structured session rollback compatibility', () => { await mkdir(journalDir, { recursive: true }) await writeFile(join(journalDir, 'journal.log'), 'durable-journal-fixture\n') - const target = await AgentSessionRecordStore.open({ directory: storeDir, hostId: 'local' }) - expect(target.getVisibleSessionTabIndex()).toEqual({ present: false, sessionIds: [] }) - await target.setSessionTabVisibility(SESSION, true) - expect(target.getVisibleSessionTabIndex()).toEqual({ present: true, sessionIds: [SESSION] }) - const targetProfile = profileWithStructuredTab() const targetParsed = safeParseWorkspaceSession(targetProfile) expect(targetParsed?.success).toBe(true) - expect( - collectSavedStructuredAgentSessionIds(targetParsed?.success ? targetParsed.data : null) - ).toEqual([SESSION]) + const targetTabs = collectSavedStructuredAgentSessionIds( + targetParsed?.success ? targetParsed.data : null + ) + expect(targetTabs).toEqual([SESSION]) + + // The store predates any tab index: it is seeded from the saved session at open (L4b). + const target = await AgentSessionRecordStore.open({ + directory: storeDir, + hostId: 'local', + savedTabSessionIds: () => targetTabs + }) + expect(target.listVisibleSessionIds()).toEqual([SESSION]) + await target.setConversationName(SESSION, 'first write persists the seed') const baseProfile = pinnedBaseRoundTrip(targetProfile) const baseParsed = safeParseWorkspaceSession(baseProfile) @@ -126,7 +131,11 @@ describe('structured session rollback compatibility', () => { targetReloadedProfile?.success ? targetReloadedProfile.data : null ) ).toEqual([]) - const reloaded = await AgentSessionRecordStore.open({ directory: storeDir, hostId: 'local' }) + const reloaded = await AgentSessionRecordStore.open({ + directory: storeDir, + hostId: 'local', + savedTabSessionIds: () => [] + }) expect(reloaded.listVisibleSessionIds()).toEqual([SESSION]) expect(reloaded.getRecord(SESSION)?.providerHandleChain).toHaveLength(1) await expect(readFile(join(journalDir, 'journal.log'), 'utf8')).resolves.toBe( @@ -138,7 +147,6 @@ describe('structured session rollback compatibility', () => { await reloaded.setSessionTabVisibility(SESSION, false) const afterClose = await AgentSessionRecordStore.open({ directory: storeDir, hostId: 'local' }) - expect(afterClose.getVisibleSessionTabIndex()).toEqual({ present: true, sessionIds: [] }) expect(afterClose.listVisibleSessionIds()).toEqual([]) expect(afterClose.getRecord(SESSION)?.providerHandleChain).toHaveLength(1) }) diff --git a/src/main/runtime/structured-agent-session-runtime-exit.test.ts b/src/main/runtime/structured-agent-session-runtime-exit.test.ts index 7a3736b9009b..6a4191fe294f 100644 --- a/src/main/runtime/structured-agent-session-runtime-exit.test.ts +++ b/src/main/runtime/structured-agent-session-runtime-exit.test.ts @@ -32,7 +32,7 @@ describe('structured session runtime provider-exit wiring', () => { } }) - it('reacquires through the production callback and accepts a distinct next message', async () => { + it('does not respawn after a provider exit, and the next message starts a new child', async () => { root = await mkdtemp(join(tmpdir(), 'orca-runtime-provider-exit-')) operations = 0 const connections: { @@ -98,16 +98,17 @@ describe('structured session runtime provider-exit wiring', () => { JSON.stringify({ refusal: attached.refusal, connections: connections.length }) ) } - await host.hold(SESSION, 'desktop-chat:1') - const exitedFence = host.deps.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 const exited = connections[0] exited?.handlers.onExit?.(new Error('scripted provider exit')) - - await vi.waitFor(() => expect(connections).toHaveLength(2)) + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + ) + // Nothing restarts the child on its own; the exit is shown and the chat waits for a send. + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(connections).toHaveLength(1) const recoveredFence = host.deps.store.getRecord(SESSION)?.lease.runtimeFence - expect(recoveredFence).toBeGreaterThan(exitedFence) if (recoveredFence === undefined) { - throw new Error('recovered lease omitted its fence') + throw new Error('released lease omitted its fence') } const body = hostTestMessage('continue with a distinct message') const envelope = { @@ -128,7 +129,9 @@ describe('structured session runtime provider-exit wiring', () => { await expect( host.send({ callerKey: 'runtime-test' }, { envelope, body }) ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) - expect(turn).toBe(1) + // The send answers at acceptance; the delivery loop starts a new child and hands it over. + await vi.waitFor(() => expect(connections).toHaveLength(2)) + await vi.waitFor(() => expect(turn).toBe(1)) }) it('does not reacquire when the production exit callback comes from a requested close', async () => { @@ -199,7 +202,6 @@ describe('structured session runtime provider-exit wiring', () => { JSON.stringify({ refusal: attached.refusal, connections: connections.length }) ) } - await host.hold(SESSION, 'desktop-chat:requested-close') await stopStructuredAgentSessionRuntime() await new Promise((resolve) => setImmediate(resolve)) @@ -222,8 +224,8 @@ describe('structured session runtime provider-exit wiring', () => { openCodexConnection: openConnection, readProcessStartTime: async () => 1_700_000_000_000 }) - await restarted.restoreReadableSessions() - const history = restarted.history({ sessionId: SESSION, direction: 'tail' }) + await restarted.restoreStartupSessions() + const history = await restarted.history({ sessionId: SESSION, direction: 'tail' }) expect(history.ok && history.page.items.some((item) => item.body.kind === 'status')).toBe(false) expect(restarted.deps.store.getRecord(SESSION)?.providerHandleChain.at(-1)?.handle).toEqual({ provider: 'codex', @@ -231,7 +233,7 @@ describe('structured session runtime provider-exit wiring', () => { }) }) - it('waits for an in-flight recovery before tearing down the runtime', async () => { + it('waits for a start a send began before tearing down the runtime', async () => { root = await mkdtemp(join(tmpdir(), 'orca-runtime-recovery-shutdown-')) let releaseRecovery!: () => void const recoveryReleased = new Promise((resolve) => { @@ -300,8 +302,28 @@ describe('structured session runtime provider-exit wiring', () => { attachParams.envelope.clientOperationId = operationId() const attached = await host.attach({ callerKey: 'runtime-test' }, attachParams) expect(attached.ok).toBe(true) - await host.hold(SESSION, 'desktop-chat:shutdown-race') - connections[0]?.handlers.onExit?.(new Error('recovery is still opening')) + connections[0]?.handlers.onExit?.(new Error('the first child exited')) + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') + ) + const body = hostTestMessage('start again') + await host.send( + { callerKey: 'runtime-test' }, + { + envelope: { + sessionId: SESSION, + clientOperationId: operationId(), + expectedRuntimeFence: host.deps.store.getRecord(SESSION)?.lease.runtimeFence ?? 0, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } + ) + // The send's start is still opening its connection when the quit begins. await vi.waitFor(() => expect(opens).toBe(2)) let stopped = false @@ -387,7 +409,6 @@ describe('structured session runtime provider-exit wiring', () => { expect(await host.attach({ callerKey: 'runtime-test' }, attachParams)).toMatchObject({ ok: true }) - await host.hold(SESSION, 'desktop-chat:backstop') await expect(stopStructuredAgentSessionRuntime()).rejects.toThrow() await new Promise((resolve) => setImmediate(resolve)) @@ -396,7 +417,7 @@ describe('structured session runtime provider-exit wiring', () => { expect(closeAttempts).toBeGreaterThanOrEqual(2) // The callback it delivered neither reacquired nor wrote a technical row. expect(connections).toHaveLength(1) - const history = host.history({ sessionId: SESSION, direction: 'tail' }) + const history = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(history.ok && history.page.items.some((item) => item.body.kind === 'status')).toBe(false) // The aborted eviction left the session reachable, so the next teardown is a real retry. diff --git a/src/main/runtime/structured-agent-session-runtime-provider-started.test.ts b/src/main/runtime/structured-agent-session-runtime-provider-started.test.ts index 602507ef4d7e..91bf91738b5b 100644 --- a/src/main/runtime/structured-agent-session-runtime-provider-started.test.ts +++ b/src/main/runtime/structured-agent-session-runtime-provider-started.test.ts @@ -1,5 +1,5 @@ -// A Claude child proving its start must not wait on another session's exit recovery, and must -// not make that recovery, or the session's own serialized operations, wait on the CLI: the host +// A Claude child proving its start must not wait on another session's start, and must not make +// that start, or the session's own serialized operations, wait on the CLI: the host // records what the child proved from what the adapter already holds. import { afterEach, describe, expect, it, vi } from 'vitest' @@ -18,7 +18,7 @@ afterEach(async () => { }) describe('a Claude child proving its start', () => { - it("never holds another session's exit recovery, or its own close, on a CLI read", async () => { + it("never holds another session's start, or its own close, on a CLI read", async () => { claude.behave(STALLED, { stallsControlReads: true }) const host = await claude.install() @@ -56,23 +56,15 @@ describe('a Claude child proving its start', () => { await vi.waitFor(() => expect(closed).toBe(true)) }) - it("flips to ready while another session's exit recovery is still acquiring", async () => { + it("flips to ready while another session's start is still acquiring", async () => { const host = await claude.install() - await expect(host.attach(CALLER, claude.attachParams(HEALTHY, null))).resolves.toMatchObject({ - ok: true - }) - await host.hold(HEALTHY, 'surface-1') - await waitForStructuredAgentSessionRecovery() - - // Its exit recovery reacquires, and that spawn never returns. The exit hands the lease back - // and the restart queued behind it reserves it again at once, so `released` is not a state a - // poll can count on seeing; `reserved` with the spawn hanging is what "still acquiring" is. + // This start never returns from its spawn, so its lease stays reserved with no child. claude.behave(HEALTHY, { spawnHangs: true }) - claude.child(HEALTHY).exit(new Error('claude stream-json exited (code 1): crashed')) + void host.attach(CALLER, claude.attachParams(HEALTHY, null)).catch(() => undefined) await vi.waitFor(() => expect(host.deps.store.getRecord(HEALTHY)?.lease.claimStatus).toBe('reserved') ) - expect(claude.children(HEALTHY)).toHaveLength(1) + const childrenWhileHung = claude.children(HEALTHY).length await expect(host.attach(CALLER, claude.attachParams(STALLED, null))).resolves.toMatchObject({ ok: true @@ -83,9 +75,9 @@ describe('a Claude child proving its start', () => { effort: 'high' }) ) - // The other recovery is still where it was: reserved, with no child yet. + // The other start is still where it was: reserved, with no new child. expect(host.deps.store.getRecord(HEALTHY)?.lease.claimStatus).toBe('reserved') - expect(claude.children(HEALTHY)).toHaveLength(1) + expect(claude.children(HEALTHY)).toHaveLength(childrenWhileHung) }) it('is drained by the runtime before teardown proceeds', async () => { diff --git a/src/main/runtime/structured-agent-session-runtime-teardown.test.ts b/src/main/runtime/structured-agent-session-runtime-teardown.test.ts new file mode 100644 index 000000000000..ff4a557b0175 --- /dev/null +++ b/src/main/runtime/structured-agent-session-runtime-teardown.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it, vi } from 'vitest' +import { tearDownRuntime, type InstalledRuntime } from './structured-agent-session-runtime-teardown' + +function installed(waitForRecovery: () => Promise) { + const savedStatus = { close: vi.fn() } + const runtime = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: teardown reads only `flushAllStreamedEvents` from the host. + host: { + flushAllStreamedEvents: vi.fn(async () => undefined) + } as unknown as InstalledRuntime['host'], + adapter: { closeAll: vi.fn(async () => undefined) }, + savedStatus, + waitForRecovery + } + return { runtime, savedStatus } +} + +describe('structured runtime teardown', () => { + it('closes the saved-status store last', async () => { + const { runtime, savedStatus } = installed(async () => undefined) + await tearDownRuntime(runtime, 'quit') + expect(savedStatus.close).toHaveBeenCalledTimes(1) + expect(savedStatus.close.mock.invocationCallOrder[0]).toBeGreaterThan( + runtime.adapter.closeAll.mock.invocationCallOrder[0]! + ) + }) + + it('still closes the saved-status store when the first recovery drain fails', async () => { + const { runtime, savedStatus } = installed(async () => { + throw new Error('recovery failed') + }) + await expect(tearDownRuntime(runtime, 'quit')).rejects.toThrow('recovery failed') + expect(savedStatus.close).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/runtime/structured-agent-session-runtime-teardown.ts b/src/main/runtime/structured-agent-session-runtime-teardown.ts index 90f4ff92c4bc..4ceb1e912f80 100644 --- a/src/main/runtime/structured-agent-session-runtime-teardown.ts +++ b/src/main/runtime/structured-agent-session-runtime-teardown.ts @@ -10,6 +10,8 @@ import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wi export type InstalledRuntime = { host: StructuredAgentSessionHost adapter: { closeAll(): Promise } + /** Flushed and closed last, so the settled statuses teardown itself wrote are saved. */ + savedStatus?: { close(): void } /** Resolves after every observed adapter exit has published, and every * recovery callback it raised has settled. */ waitForRecovery: () => Promise @@ -34,7 +36,13 @@ export async function tearDownRuntime( ): Promise { // Drain an in-flight recovery before stopping children; recovery may still // be writing lifecycle rows or acquiring a replacement child. - await installed.waitForRecovery() + try { + await installed.waitForRecovery() + } catch (error) { + // Teardown stops here, but the settled statuses already recorded still reach disk. + installed.savedStatus?.close() + throw error + } const failures: unknown[] = [] // Host teardown runs FIRST, which inverts the older order. It is what stops this host's // provider children now: it evicts each owned session through the adapter, and that eviction @@ -65,6 +73,7 @@ export async function tearDownRuntime( } catch (error) { failures.push(error) } + installed.savedStatus?.close() if (failures.length === 1) { throw failures[0] } diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index a51da3f1c3c6..c678ab49a6dc 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -19,6 +19,7 @@ import { type InstalledRuntime } from './structured-agent-session-runtime-teardown' import { AgentSessionRecoveryCapsule } from './agent-session-recovery-capsule' +import { AgentSessionSavedStatusStore } from './agent-session-saved-status-store' import { createCodexStructuredLaunchResolver } from '../codex/codex-structured-launch-resolution' import type { CodexStructuredPermissionPolicy } from '../codex/codex-structured-permission-policy' import { @@ -74,6 +75,8 @@ export type StructuredAgentSessionRuntimeDeps = { hostId: string /** Key id this host's claims are minted under. */ claimKeyId: string + /** The saved workspace session's chat tabs; seeds a store that predates any tab index. */ + savedTabSessionIds?: () => readonly string[] resolveWorkspacePath: (workspaceId: string) => Promise resolveCodexCommand?: (options?: { pathEnv?: string | null; homePath?: string }) => string resolveClaudeCommand?: () => string @@ -104,6 +107,8 @@ export type StructuredAgentSessionRuntimeDeps = { onSessionStatusChanged?: StructuredAgentSessionHostDeps['onSessionStatusChanged'] /** The agent-status store; see `StructuredAgentSessionHostDeps.statusSink`. */ statusSink?: StructuredAgentSessionHostDeps['statusSink'] + /** See `StructuredAgentSessionHostDeps.hasOpenDispatch`. */ + hasOpenDispatch?: StructuredAgentSessionHostDeps['hasOpenDispatch'] reapOrphanChildren?: typeof stopOrphanAgentSessionChildren /** The account home a structured launch would pin right now, for catalog * reads with no session record. Absent disables the catalog surface. */ @@ -195,7 +200,8 @@ async function install(deps: StructuredAgentSessionRuntimeDeps): Promise { @@ -294,10 +300,12 @@ async function install(deps: StructuredAgentSessionRuntimeDeps): Promise { await Promise.all([codex.closeAll(), claude.closeAll()]) }) + const savedStatus = AgentSessionSavedStatusStore.open(deps.stateDirectory) host = new StructuredAgentSessionHost({ store, adapter, recoveryCapsule: new AgentSessionRecoveryCapsule(deps.stateDirectory), + savedStatus, journalRoot: deps.stateDirectory, claimKeyId: deps.claimKeyId, probeOwner: createStructuredAgentSessionOwnerProbe(deps.hostId), @@ -312,12 +320,14 @@ async function install(deps: StructuredAgentSessionRuntimeDeps): Promise tab.id)) + const added: RuntimeMobileSessionAgentTab[] = [] + for (const input of tabs) { + const id = structuredAgentSessionSnapshotTabId(input.sessionId) + if (held.has(id)) { + continue + } + held.add(id) + added.push({ + type: 'agent-session', + id, + title: defaultAgentChatLabel(input.agent), + sessionId: input.sessionId, + ...(input.replacesSessionId ? { replacesSessionId: input.replacesSessionId } : {}), + agent: input.agent, + isActive: false + }) + } + const last = added.at(-1) + if (!last) { + return null + } + const activeId = options.activate ? last.id : null + if (activeId) { + last.isActive = true + } + const priorGroups = existing?.tabGroups ?? [ + { + id: getHeadlessMobileSessionGroupId(workspaceId), + activeTabId: existing?.activeTabId ?? null, + tabOrder: [] + } + ] + const groupId = + priorGroups.find((group) => group.id === existing?.activeGroupId)?.id ?? priorGroups[0]!.id + return { + worktree: workspaceId, + publicationEpoch: existing?.publicationEpoch ?? `structured:${Date.now().toString(36)}`, + snapshotVersion: (existing?.snapshotVersion ?? 0) + 1, + activeGroupId: activeId ? groupId : (existing?.activeGroupId ?? groupId), + activeTabId: activeId ?? existing?.activeTabId ?? null, + activeTabType: activeId ? 'agent-session' : (existing?.activeTabType ?? null), + tabGroups: priorGroups.map((group) => + group.id === groupId + ? { + ...group, + activeTabId: activeId ?? group.activeTabId, + tabOrder: [...group.tabOrder, ...added.map((tab) => tab.id)] + } + : group + ), + ...(existing?.tabGroupLayout ? { tabGroupLayout: existing.tabGroupLayout } : {}), + tabs: [ + ...(existing?.tabs ?? []).map((tab) => (activeId ? { ...tab, isActive: false } : tab)), + ...added + ] + } +} diff --git a/src/main/runtime/structured-agent-session-unpublished-tab-close.test.ts b/src/main/runtime/structured-agent-session-unpublished-tab-close.test.ts new file mode 100644 index 000000000000..77c30d4a5536 --- /dev/null +++ b/src/main/runtime/structured-agent-session-unpublished-tab-close.test.ts @@ -0,0 +1,73 @@ +// A restored chat the renderer shows from its saved session before the host has listed it: the +// user can close it, and that close has to reach the durable index or the next listing revives it. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../shared/agent-session-record.test-fixture' +import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { OrcaRuntimeService } from './orca-runtime' + +afterEach(() => setStructuredAgentSessionHost(null)) + +type RuntimeInternals = { + hasPersistedStructuredAgentSessionStore(): boolean + ensureStructuredAgentSessionHost(): Promise +} + +/** A host installed only when something asks for it, with one chat record in `workspace-1`. */ +function runtimeInstallingHost(listed: readonly string[]) { + const setSessionTabVisibility = vi.fn(async () => undefined) + const close = vi.fn(async () => undefined) + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as RuntimeInternals + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.ensureStructuredAgentSessionHost = async () => { + const record = agentSessionRecordFixture(agentSessionLeaseFixture({ sessionId: 'kept' })) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test reads only the host members stubbed here. + setStructuredAgentSessionHost({ + deps: { + store: { + getRecord: (sessionId: string) => + sessionId === 'kept' + ? { ...record, location: { ...record.location, workspaceId: 'workspace-1' } } + : null + } + }, + listVisibleSessionIds: () => [...listed], + setSessionTabVisibility, + close + } as never) + } + return { runtime, setSessionTabVisibility, close } +} + +describe('closing a restored chat before its tab is listed', () => { + it('installs the host and retires the chat from the durable index', async () => { + const { runtime, setSessionTabVisibility, close } = runtimeInstallingHost(['kept']) + + await runtime.closeMobileSessionTab('id:workspace-1', 'agent-session:kept', { reason: 'user' }) + + expect(setSessionTabVisibility).toHaveBeenCalledWith('kept', false) + expect(close).toHaveBeenCalledWith('kept') + }) + + it.each([ + ['names another workspace', 'workspace-2', ['kept']], + ['addresses a chat whose tab is already closed', 'workspace-1', []] + ])( + 'refuses an unlisted chat close that %s, as for any unknown tab', + async (_case, worktreeId, listed) => { + const { runtime, setSessionTabVisibility, close } = runtimeInstallingHost(listed) + + await expect( + runtime.closeMobileSessionTab(`id:${worktreeId}`, 'agent-session:kept', { reason: 'user' }) + ).rejects.toThrow('tab_not_found') + + expect(setSessionTabVisibility).not.toHaveBeenCalled() + expect(close).not.toHaveBeenCalled() + } + ) +}) diff --git a/src/main/runtime/structured-session-worktree-teardown.test.ts b/src/main/runtime/structured-session-worktree-teardown.test.ts index 602aaa3d3aa0..c46ba8575f80 100644 --- a/src/main/runtime/structured-session-worktree-teardown.test.ts +++ b/src/main/runtime/structured-session-worktree-teardown.test.ts @@ -97,7 +97,7 @@ function installHost(options: { } }, hasSession: (sessionId: string) => held.has(sessionId), - getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: [...visible] }), + listVisibleSessionIds: () => [...visible], setSessionTabVisibility: async (sessionId: string, isVisible: boolean) => { if (!isVisible) { visible.delete(sessionId) diff --git a/src/main/runtime/structured-session-worktree-teardown.ts b/src/main/runtime/structured-session-worktree-teardown.ts index f79d5d128128..3f8ab31b3ec7 100644 --- a/src/main/runtime/structured-session-worktree-teardown.ts +++ b/src/main/runtime/structured-session-worktree-teardown.ts @@ -27,7 +27,10 @@ import { } from '../../shared/execution-host' import { STILL_LIVE_DETAIL_PREFIX } from '../../shared/worktree/removal' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' -import { observeStructuredWorker } from './structured-worker-authority' +import { + observeStructuredWorker, + structuredSessionCloseSettled +} from './structured-worker-authority' import { closeStructuredAgentSessionChild } from './structured-agent-session-close' import { retireSettledStructuredWorkerTab } from './structured-agent-session-tab-retirement' import type { WorktreePtyHostFence } from './worktree-pty-host-fence' @@ -86,12 +89,9 @@ export function structuredSessionTeardownHostId( * The workspace's structured sessions, split into what belongs to it and what is attached. * * MEMBERSHIP and LIVENESS answer different questions, and folding them into one list is what let - * a chat tab outlive its workspace. A provider child is scoped to a VISIBLE pane — the hold that - * keeps one is `enabled: isVisible && isWorktreeActive`, and dropping the last hold evicts the - * child after the release grace — so `live` really means "this chat is the visible pane in the - * active workspace, or was moments ago". Deleting a workspace from the sidebar while a different - * one is active makes every chat in the target non-live. Those are exactly the sessions a - * liveness-only list never saw. + * a chat tab outlive its workspace. A provider child runs from a send until the idle sweep rests + * it, so `live` only means "this chat's agent worked recently", and every chat at rest in the + * target is non-live. Those are exactly the sessions a liveness-only list never saw. */ export type StructuredSessionsForWorktree = { /** Every session bound to this workspace on the fenced host, attached or not. */ @@ -256,11 +256,10 @@ export async function closeStructuredSessionsForWorktree( } = {} ): Promise { const { runtime, mayRefuse } = options - // No `afterClose` for a dispatched worker: `host.close` drops the holds, so nothing keeps a - // provider child un-evictable, but the dispatch's redrive subscription and registry entry do - // survive until it settles by another verb. That is a bounded leak, not a hazard — and passing - // one here would mean resolving a dispatch id per session on a teardown path that must stay - // inside the sweep deadline. + // No `afterClose` for a dispatched worker: `host.close` stops the child, but the dispatch's + // redrive subscription and registry entry survive until it settles by another verb. That is a + // bounded leak, not a hazard — and passing one here would mean resolving a dispatch id per + // session on a teardown path that must stay inside the sweep deadline. for (const session of progress.sessions) { // Stops ISSUING new closes once the budget is spent; an in-flight one is left to finish, since // nothing here can cancel a provider round trip. Without this, one slow round trip starved @@ -279,7 +278,7 @@ export async function closeStructuredSessionsForWorktree( // Re-observed rather than reusing the close's own reason string: what the user is asked to // waive is the state AFTER the attempt, and a close that threw never reached an observation. const status = observeStructuredWorker({ sessionId: session.sessionId }).status - if (status === 'exited') { + if (status === 'exited' || structuredSessionCloseSettled(session.sessionId)) { // The re-read can PROVE the exit a failed close could not — it threw past its own // observation, or the record's death evidence landed after it read. Refusing on a child // that is demonstrably gone is the defect this sweep exists to remove, so take the proof diff --git a/src/main/runtime/structured-worker-at-rest.test.ts b/src/main/runtime/structured-worker-at-rest.test.ts new file mode 100644 index 000000000000..5ecab3a4e440 --- /dev/null +++ b/src/main/runtime/structured-worker-at-rest.test.ts @@ -0,0 +1,347 @@ +// A structured worker whose agent is at rest: its dispatch keeps it running while open, and once it +// rests it is still this runtime's worker — mail reaches it — until its chat tab is gone. Whether +// its process runs is a separate answer, and a close counts a released lease as done. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecord } from '../../shared/agent-session-record' +import { OrchestrationDb } from './orchestration/db' + +const hostRef: { current: unknown } = { current: null } + +vi.mock('../native-chat/agent-session-wire/structured-agent-session-registry', () => ({ + getStructuredAgentSessionHost: () => hostRef.current +})) + +const { observeStructuredWorker, resolveStructuredWorkerAuthority, structuredSessionCloseSettled } = + await import('./structured-worker-authority') +const { structuredWorkerOwesWork, structuredWorkerOwned } = + await import('./structured-worker-custody') +const { + mintStructuredWorkerHandle, + mintStructuredWorkerPaneKey, + structuredWorkerIdentities, + structuredWorkerProcessIncarnation +} = await import('./structured-worker-identity') +const { listAddressableStructuredWorkers } = + await import('./orchestration/structured-worker-group-addressing') +const { closeStructuredAgentSessionChild } = await import('./structured-agent-session-close') +const { resolveGroupAddress } = await import('./orchestration/groups') +const { createRestTestRig, foundRestTestChat, IDLE_MS, REST_TEST_SESSION, sweepTicks } = + await import('../native-chat/agent-session-wire/structured-agent-session-rest-test-rig') + +const SESSION = 'a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d' +const LOCAL = JSON.stringify({ kind: 'local', hostId: 'local' }) + +function record(lease: Partial): AgentSessionRecord { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a partial record; the ownership and liveness reads touch only lease, location and provider. + return { + sessionId: SESSION, + provider: 'codex', + location: { + executionHostId: 'local', + wslDistro: null, + workspaceId: 'wt_1', + workspaceKind: 'git-worktree' + }, + lease: { claimStatus: 'live', deathEvidence: null, runtimeFence: 3, ...lease } + } as AgentSessionRecord +} + +function installHost(current: AgentSessionRecord | null, tabs: string[]) { + const host = { + deps: { store: { getRecord: () => current } }, + hasSession: () => false, + listVisibleSessionIds: () => tabs, + setSessionTabVisibility: vi.fn(async (_id: string, visible: boolean) => { + tabs.splice(0, tabs.length, ...(visible ? [SESSION] : [])) + }), + close: vi.fn(async () => undefined) + } + hostRef.current = host + return host +} + +let paneKey = '' + +function registerWorker(): string { + const handle = mintStructuredWorkerHandle() + paneKey = mintStructuredWorkerPaneKey(SESSION) + structuredWorkerIdentities.register({ + handle, + sessionId: SESSION, + agent: 'codex', + paneKey, + processIncarnation: structuredWorkerProcessIncarnation(SESSION), + worktreeId: 'wt_1', + hostScope: { kind: 'local', hostId: 'local' } + }) + return handle +} + +beforeEach(() => { + structuredWorkerIdentities.clear() + hostRef.current = null +}) + +describe('an open dispatch keeps its worker running (P2-19 i)', () => { + let db: OrchestrationDb + beforeEach(() => { + db = new OrchestrationDb(':memory:') + }) + afterEach(() => db.close()) + + function dispatchWorker(hostScope = LOCAL) { + const task = db.createTask({ runId: 'run_legacy_local', spec: 'structured work' }) + const { dispatch } = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 9 + }) + db.prepareStartingWorkerAuthority({ + dispatchId: dispatch.id, + handle: mintStructuredWorkerHandle(), + paneKey: mintStructuredWorkerPaneKey(SESSION), + processIncarnation: structuredWorkerProcessIncarnation(SESSION), + worktreeId: 'wt_1', + effects: [], + setupState: 'not_configured', + hostScope, + terminalOwnership: 'created' + }) + return dispatch + } + + it('answers open while the dispatch starts, runs or stops, and closed once it settles', () => { + const dispatch = dispatchWorker() + expect(structuredWorkerOwesWork(db, record({}))).toBe(true) + db.markWorkerDispatchReady(dispatch.id) + expect(structuredWorkerOwesWork(db, record({}))).toBe(true) + db.beginWorkerStop(dispatch.id, 'epoch_home') + expect(structuredWorkerOwesWork(db, record({}))).toBe(true) + db.settleWorkerStop(dispatch.id) + expect(structuredWorkerOwesWork(db, record({}))).toBe(false) + }) + + // Custody is the list state coordinators see: `active` owes work, `reclaimable` does not. + it('lets a worker awaiting its coordinator rest, and keeps one whose stop is in doubt', () => { + const done = dispatchWorker() + db.markWorkerDispatchReady(done.id) + db.settleWorkerReport({ + taskId: done.task_id, + dispatchId: done.id, + outcome: 'succeeded', + result: 'done' + }) + expect(structuredWorkerOwesWork(db, record({}))).toBe(false) + + const doubted = dispatchWorker() + db.markWorkerDispatchReady(doubted.id) + db.beginWorkerStop(doubted.id, 'epoch_home') + db.markWorkerStopUnknown(doubted.id, 'the close was not proven') + expect(structuredWorkerOwesWork(db, record({}))).toBe(true) + }) + + it('keeps a worker at rest after its dispatch settles a group recipient, until its tab goes (P2-19 ii)', () => { + const dispatch = dispatchWorker() + db.markWorkerDispatchReady(dispatch.id) + db.beginWorkerStop(dispatch.id, 'epoch_home') + db.settleWorkerStop(dispatch.id) + // Settlement forgets the in-memory entry; only the durable row and the record remain. + structuredWorkerIdentities.clear() + const tabs = [SESSION] + installHost( + record({ + claimStatus: 'released', + deathEvidence: { kind: 'exit-observed', detail: 'stopped by the sweep', observedAt: 1 } + }), + tabs + ) + const handle = db.getWorkerTerminalResourceByOwner(dispatch.id)!.terminal_handle + const recipients = () => listAddressableStructuredWorkers(db) + + expect(recipients()).toEqual([{ handle, worktreeId: 'wt_1', agentIdentity: 'codex' }]) + expect(resolveGroupAddress('@codex', 'term_sender', recipients(), () => 'idle')).toEqual([ + handle + ]) + expect(resolveGroupAddress('@claude', 'term_sender', recipients(), () => 'idle')).toEqual([]) + // Direct mail resolves the same worker through the same ownership answer. + expect(resolveStructuredWorkerAuthority(handle, db)?.identity.handle).toBe(handle) + + tabs.length = 0 + expect(recipients()).toEqual([]) + expect(resolveStructuredWorkerAuthority(handle, db)).toBeNull() + }) + + it('stops routing to a worker its coordinator abandoned and then released, and keeps its tab', () => { + const dispatch = dispatchWorker() + db.markWorkerDispatchReady(dispatch.id) + structuredWorkerIdentities.clear() + const tabs = [SESSION] + installHost( + record({ + claimStatus: 'released', + deathEvidence: { kind: 'exit-observed', detail: 'stopped by the sweep', observedAt: 1 } + }), + tabs + ) + const resource = db.getWorkerTerminalResourceByOwner(dispatch.id)! + const handle = resource.terminal_handle + const recipients = () => listAddressableStructuredWorkers(db) + // At rest, its dispatch abandoned: still a recipient, as a terminal worker left running is. + db.abandonWorkerDispatch(dispatch.id) + const worktreeGroup = () => + resolveGroupAddress('@worktree:wt_1', 'term_sender', recipients(), () => 'idle') + expect(worktreeGroup()).toEqual([handle]) + expect(resolveStructuredWorkerAuthority(handle, db)).not.toBeNull() + + // The release finds the agent at rest, so it settles as released. + expect(db.requestWorkerTerminalRelease(dispatch.id)).toMatchObject({ disposition: 'retained' }) + expect( + db.settleDeadWorkerTerminalRelease({ + requestingDispatchId: dispatch.id, + resourceId: resource.id, + processIncarnation: resource.process_incarnation! + }) + ).toMatchObject({ disposition: 'released' }) + + expect(tabs).toEqual([SESSION]) + expect(worktreeGroup()).toEqual([]) + // Direct mail routes through the same answer. + expect(resolveStructuredWorkerAuthority(handle, db)).toBeNull() + }) + + it('reads only this host scope, and no database answers no', () => { + dispatchWorker(JSON.stringify({ kind: 'ssh', targetId: 'elsewhere' })) + expect(structuredWorkerOwesWork(db, record({}))).toBe(false) + expect(structuredWorkerOwesWork(null, record({}))).toBe(false) + }) +}) + +describe('ownership, not liveness (P2-19 ii-iv, P2-26)', () => { + it('owns a worker at rest while its tab is listed, and routes mail to it', () => { + const handle = registerWorker() + installHost( + record({ + claimStatus: 'released', + deathEvidence: { kind: 'exit-observed', detail: 'stopped', observedAt: 1 } + }), + [SESSION] + ) + + expect(structuredWorkerOwned(SESSION)).toBe(true) + expect(observeStructuredWorker({ sessionId: SESSION }).status).toBe('exited') + expect(resolveStructuredWorkerAuthority(handle, null)?.identity.paneKey).toBe(paneKey) + }) + + it('retires a released worker whose tab is gone', () => { + const handle = registerWorker() + installHost(record({ claimStatus: 'released' }), []) + + expect(structuredWorkerOwned(SESSION)).toBe(false) + expect(resolveStructuredWorkerAuthority(handle, null)).toBeNull() + }) + + it('keeps authority for a live session that has no tab', () => { + const handle = registerWorker() + installHost(record({ claimStatus: 'live' }), []) + + expect(structuredWorkerOwned(SESSION)).toBe(true) + expect(resolveStructuredWorkerAuthority(handle, null)).not.toBeNull() + }) + + it('cannot answer without a host', () => { + expect(structuredWorkerOwned(SESSION)).toBeNull() + }) +}) + +describe('a close whose stop could not be proven (P2-30)', () => { + it('closes on the first attempt, keeps the tab retired and leaves the verdict alone', async () => { + const released = record({ claimStatus: 'released', deathEvidence: null }) + const tabs = [SESSION] + const host = installHost(released, tabs) + expect(observeStructuredWorker({ sessionId: SESSION }).status).toBe('unverifiable') + + const outcome = await closeStructuredAgentSessionChild(SESSION) + expect(outcome).toMatchObject({ stopped: true, closeAttempted: true }) + expect(host.close).toHaveBeenCalledOnce() + expect(tabs).toEqual([]) + expect(host.setSessionTabVisibility).not.toHaveBeenCalledWith(SESSION, true) + expect(released.lease.deathEvidence).toBeNull() + expect(structuredSessionCloseSettled(SESSION)).toBe(true) + }) + + it('still refuses a close that left the lease live', () => { + installHost(record({ claimStatus: 'live' }), [SESSION]) + expect(structuredSessionCloseSettled(SESSION)).toBe(false) + }) +}) + +describe('a task dispatched into a worker whose own dispatch settled', () => { + it('keeps the worker running while that task is open, and lets it rest once the task settles', async () => { + const db = new OrchestrationDb(':memory:') + const rig = await createRestTestRig({ + hasOpenDispatch: (current) => structuredWorkerOwesWork(db, current) + }) + try { + hostRef.current = rig.host + await foundRestTestChat(rig) + const incarnation = structuredWorkerProcessIncarnation(REST_TEST_SESSION) + const handle = mintStructuredWorkerHandle() + const workerPane = mintStructuredWorkerPaneKey(REST_TEST_SESSION) + const first = db.createTask({ runId: 'run_legacy_local', spec: 'first task' }) + const { dispatch: started } = db.createStartingWorkerDispatch({ + taskId: first.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 9 + }) + db.prepareStartingWorkerAuthority({ + dispatchId: started.id, + handle, + paneKey: workerPane, + processIncarnation: incarnation, + worktreeId: 'wt_1', + effects: [], + setupState: 'not_configured', + hostScope: LOCAL, + terminalOwnership: 'created' + }) + db.markWorkerDispatchReady(started.id) + expect( + db.settleWorkerReport({ + taskId: first.id, + dispatchId: started.id, + outcome: 'succeeded', + result: 'done' + }) + ).toMatchObject({ action: 'settled' }) + // The coordinator hands the same worker its next task: a dispatch with no worker row. + const second = db.createTask({ runId: 'run_legacy_local', spec: 'second task' }) + const handedOn = db.createDispatchContext({ + taskId: second.id, + assigneeHandle: handle, + assigneePaneKey: workerPane, + processIncarnation: incarnation, + creator: { kind: 'system' }, + maxDepth: 9 + }) + rig.clock.now += 2 * IDLE_MS + + await sweepTicks(12) + expect(rig.adapter.closeSession).not.toHaveBeenCalled() + expect(observeStructuredWorker({ sessionId: REST_TEST_SESSION }).status).toBe('live') + + db.completeDispatch(handedOn.id) + rig.clock.now += IDLE_MS + 1 + await vi.waitFor(() => + expect(observeStructuredWorker({ sessionId: REST_TEST_SESSION }).status).toBe('exited') + ) + expect(rig.adapter.closeSession).toHaveBeenCalledWith(REST_TEST_SESSION) + } finally { + hostRef.current = null + await rig.dispose() + db.close() + } + }) +}) diff --git a/src/main/runtime/structured-worker-authority.ts b/src/main/runtime/structured-worker-authority.ts index 96d013c2ed3f..24fda6b1456e 100644 --- a/src/main/runtime/structured-worker-authority.ts +++ b/src/main/runtime/structured-worker-authority.ts @@ -3,8 +3,10 @@ * * The registry holds the handle→session mapping for this process; the durable worker-terminal * resource row is what survives a restart, so a miss falls back to rehydrating from it. The - * durable agent-session record is the liveness half: a session whose claim is conflicted or - * released, or pinned to another execution host, is no longer this runtime's structured worker. + * durable agent-session record, the chat's tab and the orchestration's own resource row decide + * custody: see `structured-worker-custody`. + * Whether its provider process runs is a separate fact, `observeStructuredWorker`, and routing + * never reads it — an agent at rest still receives mail, which starts it. */ import type { AgentSessionRecord } from '../../shared/agent-session-record' @@ -12,11 +14,11 @@ import type { OrcaSessionId } from '../../shared/orca-session-address' import type { RuntimeTerminalState } from '../../shared/runtime-types' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import type { OrchestrationDb } from './orchestration/db' +import { structuredWorkerAddressable } from './structured-worker-custody' import { isStructuredWorkerHandle, structuredWorkerIdentities, structuredWorkerProcessIncarnation, - structuredWorkerRecordIsCurrent, type StructuredWorkerIdentity } from './structured-worker-identity' @@ -82,7 +84,8 @@ export function isRecordedStructuredWorkerSession( ) } -/** Identity plus a record that still proves this runtime owns the session. */ +/** Identity plus a record that still proves this runtime owns the session, for a worker its + * orchestration has not released. */ export function resolveStructuredWorkerAuthority( handle: string, db: OrchestrationDb | null | undefined @@ -92,7 +95,14 @@ export function resolveStructuredWorkerAuthority( return null } const record = readStructuredAgentSessionRecord(identity.sessionId) - return record && structuredWorkerRecordIsCurrent(record) ? { identity, record } : null + return record && + structuredWorkerAddressable( + db, + identity.sessionId, + db?.getWorkerTerminalResourceByHandle?.(identity.handle) + ) + ? { identity, record } + : null } /** @@ -115,6 +125,20 @@ export type StructuredWorkerObservation = { reason?: string } +/** + * Whether a close left nothing running: `exited`, or `unverifiable` on a released lease — a release + * whose stop could not be proven, which sent no signal and is left as it is. Closing a chat is the + * user's action, and bookkeeping about a process already released must not refuse it. + */ +export function structuredSessionCloseSettled(sessionId: string): boolean { + const status = observeStructuredWorker({ sessionId }).status + return ( + status === 'exited' || + (status === 'unverifiable' && + readStructuredAgentSessionRecord(sessionId)?.lease.claimStatus === 'released') + ) +} + /** * The observation as the terminal state every read result reports. * diff --git a/src/main/runtime/structured-worker-custody.ts b/src/main/runtime/structured-worker-custody.ts new file mode 100644 index 000000000000..3c90761c3860 --- /dev/null +++ b/src/main/runtime/structured-worker-custody.ts @@ -0,0 +1,133 @@ +/** + * Whether orchestration still holds a structured worker, the one answer every reader derives from. + * + * Custody is the orchestration's own worker-terminal state, the list state `worker-list` shows + * coordinators, never whether the worker's process runs: a worker at rest is still held, and mail + * starts it. Routing, group addressing and `worker-show` ask whether it is addressable; the idle + * sweep asks whether it still owes work. + * + * Two policy decisions live here and nowhere else: only `released` ends addressability (a release + * pending or in doubt still routes, as for a terminal worker), and a settled worker awaiting its + * coordinator's decision (`reclaimable`) owes no work, so it may rest. + */ + +import type { AgentSessionRecord } from '../../shared/agent-session-record' +import type { OrchestrationDb } from './orchestration/db' +import type { WorkerDispatchState } from './orchestration/types' +import { + deriveWorkerTerminalListState, + type WorkerTerminalResourceRow +} from './orchestration/worker-terminal-ownership' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { + structuredWorkerHostScope, + structuredWorkerProcessIncarnation, + structuredWorkerRecordIsCurrent +} from './structured-worker-identity' + +/** + * Whether this runtime still owns the worker's session: routing, addressing and authority ask + * this, never whether its process runs. Null when the host is not installed, because reading the + * record store would install it — not being able to look is not an answer. + */ +export function structuredWorkerOwned(sessionId: string): boolean | null { + const host = getStructuredAgentSessionHost() + if (!host) { + return null + } + let record: AgentSessionRecord | null + try { + record = host.deps.store.getRecord(sessionId) + } catch { + record = null + } + return structuredWorkerRecordIsCurrent( + record, + record?.lease.claimStatus === 'released' && structuredWorkerTabListed(host, sessionId) + ) +} + +/** Retirement is the tab index: every path that ends a chat for good hides its tab. */ +function structuredWorkerTabListed( + host: NonNullable>, + sessionId: string +): boolean { + try { + return host.listVisibleSessionIds?.().includes(sessionId) ?? false + } catch { + return false + } +} + +type CustodyRow = Pick< + WorkerTerminalResourceRow, + 'owner_dispatch_id' | 'terminal_handle' | 'ownership_state' | 'release_state' +> + +function ownerState( + db: OrchestrationDb | null | undefined, + row: CustodyRow +): WorkerDispatchState | undefined { + return ( + db?.getWorkerDispatch?.(row.owner_dispatch_id)?.state ?? + db?.getRemoteDispatchAttachment?.(row.owner_dispatch_id)?.state + ) +} + +/** + * The user still owns the chat and orchestration has not released the worker, as with a terminal + * worker whose terminal closed. Null when ownership cannot be read. A released worker's chat stays + * the user's; nothing routes to it. + */ +export function structuredWorkerAddressable( + db: OrchestrationDb | null | undefined, + sessionId: string, + row: CustodyRow | undefined +): boolean | null { + const owned = structuredWorkerOwned(sessionId) + if (owned === null) { + return null + } + // Release is read off the row alone, so an owner whose state is unreadable still answers. + const custody = row + ? deriveWorkerTerminalListState({ + workerState: ownerState(db, row) ?? 'unsupervised', + agentTerminalHandle: row.terminal_handle, + resource: row + }) + : null + return owned && custody !== 'released' +} + +/** + * Work orchestration still owes on this worker, read per sweep tick: any unsettled dispatch + * addressed to its incarnation, on a process this host owns a terminal for. That covers its own + * worker-start dispatch (whose context stays open while the worker is active, a stop in doubt + * included, because a supervised worker's context settles only with it) and any task later + * dispatched to it. A `reclaimable` worker's dispatch has settled, so it owes nothing. + */ +export function structuredWorkerOwesWork( + db: OrchestrationDb | null, + record: AgentSessionRecord +): boolean { + const hostScope = structuredWorkerHostScope(record.location) + if (!db || !hostScope) { + return false + } + const incarnation = structuredWorkerProcessIncarnation(record.sessionId) + const owned = db.db + .prepare( + `SELECT 1 FROM worker_terminal_resources + WHERE process_incarnation = ? AND host_scope IS ? AND ownership_state = 'owned' LIMIT 1` + ) + .get(incarnation, JSON.stringify(hostScope)) + return ( + owned !== undefined && + db.db + .prepare( + `SELECT 1 FROM dispatch_contexts + WHERE process_incarnation = ? AND status IN ('pending', 'dispatched') LIMIT 1` + ) + .get(incarnation) !== undefined + ) +} diff --git a/src/main/runtime/structured-worker-identity.test.ts b/src/main/runtime/structured-worker-identity.test.ts index 98989a85891c..d20cadf5c770 100644 --- a/src/main/runtime/structured-worker-identity.test.ts +++ b/src/main/runtime/structured-worker-identity.test.ts @@ -176,18 +176,24 @@ describe('structured worker identity', () => { it('keeps a recovered session current across a fence bump', () => { // The host bumps the fence on its own transparent crash recovery; fencing identity on it // would wedge the SAME worker as identity_unproven forever. - expect(structuredWorkerRecordIsCurrent(record({ runtimeFence: 1 }))).toBe(true) - expect(structuredWorkerRecordIsCurrent(record({ runtimeFence: 9 }))).toBe(true) + expect(structuredWorkerRecordIsCurrent(record({ runtimeFence: 1 }), false)).toBe(true) + expect(structuredWorkerRecordIsCurrent(record({ runtimeFence: 9 }), false)).toBe(true) expect(structuredWorkerProcessIncarnation(SESSION_ID)).toBe( structuredWorkerProcessIncarnation(SESSION_ID) ) }) - it('refuses a conflicted or released session', () => { + it('refuses a conflicted session, and a released one whose chat tab is gone', () => { // A terminal owner an older build recorded loads conflicted; it is not this worker. - expect(structuredWorkerRecordIsCurrent(record({ claimStatus: 'conflicted' }))).toBe(false) - expect(structuredWorkerRecordIsCurrent(record({ claimStatus: 'released' }))).toBe(false) - expect(structuredWorkerRecordIsCurrent(null)).toBe(false) + expect(structuredWorkerRecordIsCurrent(record({ claimStatus: 'conflicted' }), true)).toBe(false) + expect(structuredWorkerRecordIsCurrent(record({ claimStatus: 'released' }), false)).toBe(false) + expect(structuredWorkerRecordIsCurrent(null, true)).toBe(false) + }) + + it('keeps a released session with a listed tab: that worker is at rest, not gone', () => { + expect(structuredWorkerRecordIsCurrent(record({ claimStatus: 'released' }), true)).toBe(true) + // A live lease is owned whether or not a tab shows it. + expect(structuredWorkerRecordIsCurrent(record({ claimStatus: 'live' }), false)).toBe(true) }) }) diff --git a/src/main/runtime/structured-worker-identity.ts b/src/main/runtime/structured-worker-identity.ts index c32d6c37f156..e09adee7c1e2 100644 --- a/src/main/runtime/structured-worker-identity.ts +++ b/src/main/runtime/structured-worker-identity.ts @@ -141,19 +141,62 @@ export function structuredWorkerHostScope( : null } -/** Whether the durable record still describes THIS worker under this host. */ +/** + * Whether the durable record still describes THIS worker under this host — ownership, not whether + * its process runs. A released lease is a worker at rest while its chat tab is listed; released + * with the tab gone is retired. `tabListed` is the persisted tab index's answer. + */ export function structuredWorkerRecordIsCurrent( - record: AgentSessionRecord | null | undefined + record: AgentSessionRecord | null | undefined, + tabListed: boolean ): boolean { return Boolean( record && // Why: a conflicted claim may name a terminal an older build recorded as owner, not this worker. record.lease.claimStatus !== 'conflicted' && - record.lease.claimStatus !== 'released' && + (record.lease.claimStatus !== 'released' || tabListed) && structuredWorkerHostScope(record.location) ) } +type StructuredWorkerResourceRow = { + terminal_handle: string + pane_key: string | null + process_incarnation: string | null + worktree_id: string | null + host_scope: string | null +} + +/** A worker's identity as its durable worker-terminal resource row records it, or null for a row + * that is not a structured worker's or whose pane key does not belong to its own session. */ +export function structuredWorkerIdentityFromRow( + row: StructuredWorkerResourceRow +): StructuredWorkerIdentity | null { + const sessionId = sessionIdFromStructuredWorkerIncarnation(row.process_incarnation) + const hostScope = parseWorkerTerminalHostScope(row.host_scope) + const paneKey = row.pane_key + if ( + !sessionId || + !hostScope || + !row.worktree_id || + !paneKey || + !isStructuredWorkerHandle(row.terminal_handle) || + !persistedStructuredWorkerPaneKeyIsValid(paneKey, sessionId) + ) { + return null + } + return { + handle: row.terminal_handle, + sessionId, + // The row does not carry the provider; callers that need it read the durable record. + agent: null, + paneKey, + processIncarnation: structuredWorkerProcessIncarnation(sessionId), + worktreeId: row.worktree_id, + hostScope + } +} + export class StructuredWorkerIdentityRegistry { private readonly byHandle = new Map() private readonly bySessionId = new Map() @@ -193,35 +236,9 @@ export class StructuredWorkerIdentityRegistry { * only place a structured worker's pane key and host scope outlive this process. A row whose * pane key does not belong to its own recorded session is refused rather than trusted. */ - rehydrate(row: { - terminal_handle: string - pane_key: string | null - process_incarnation: string | null - worktree_id: string | null - host_scope: string | null - }): StructuredWorkerIdentity | null { - const sessionId = sessionIdFromStructuredWorkerIncarnation(row.process_incarnation) - const hostScope = parseWorkerTerminalHostScope(row.host_scope) - if ( - !sessionId || - !hostScope || - !row.worktree_id || - !isStructuredWorkerHandle(row.terminal_handle) || - // The durable row bootstraps the registry after restart, so validate it before registration. - !persistedStructuredWorkerPaneKeyIsValid(row.pane_key, sessionId) - ) { - return null - } - return this.register({ - handle: row.terminal_handle, - sessionId, - // The row does not carry the provider; callers that need it read the durable record. - agent: null, - paneKey: row.pane_key, - processIncarnation: structuredWorkerProcessIncarnation(sessionId), - worktreeId: row.worktree_id, - hostScope - }) + rehydrate(row: StructuredWorkerResourceRow): StructuredWorkerIdentity | null { + const identity = structuredWorkerIdentityFromRow(row) + return identity ? this.register(identity) : null } clear(): void { diff --git a/src/main/runtime/structured-worker-mail-routing.test.ts b/src/main/runtime/structured-worker-mail-routing.test.ts index c426132b8485..6f10b29a87c0 100644 --- a/src/main/runtime/structured-worker-mail-routing.test.ts +++ b/src/main/runtime/structured-worker-mail-routing.test.ts @@ -20,9 +20,13 @@ const prototype = OrcaRuntimeWithAdoptTerminalOrphansFromInventory.prototype const getLivePaneKey = prototype.getLiveTerminalPaneKey const resolveActiveTerminal = prototype.resolveActiveTerminal -function installRecord(lease: { runtimeKind: string; claimStatus: string } | null): void { +function installRecord( + lease: { runtimeKind: string; claimStatus: string } | null, + tabListed = false +): void { hostRef.current = lease ? { + listVisibleSessionIds: () => (tabListed ? [SESSION_ID] : []), deps: { store: { getRecord: () => ({ @@ -73,10 +77,19 @@ describe('bare-handle direct mail to a structured session', () => { ) }) - it('withholds the pane key when the session is not proven live', () => { - // The PTY branch is connected-gated so mail is never routed to a corpse; so is this one. + it('routes to a worker at rest, whose agent the mail starts', () => { + // Released by the idle sweep with its chat tab still listed: owned, so mail reaches it. const handle = registerWorker() - installRecord({ runtimeKind: 'native', claimStatus: 'reserved' }) + installRecord({ runtimeKind: 'native', claimStatus: 'released' }, true) + expect(getLivePaneKey.call(paneKeyStub, handle)).toBe( + structuredWorkerIdentities.get(handle)!.paneKey + ) + }) + + it('withholds the pane key from a retired worker: released, with its tab gone', () => { + // Mail is never routed to a worker this runtime no longer owns. + const handle = registerWorker() + installRecord({ runtimeKind: 'native', claimStatus: 'released' }, false) expect(getLivePaneKey.call(paneKeyStub, handle)).toBeNull() }) diff --git a/src/main/runtime/structured-worker-terminal-read.test.ts b/src/main/runtime/structured-worker-terminal-read.test.ts index 802b23733bca..e49f24ba1a15 100644 --- a/src/main/runtime/structured-worker-terminal-read.test.ts +++ b/src/main/runtime/structured-worker-terminal-read.test.ts @@ -75,22 +75,22 @@ describe('reading a structured worker through the terminal-read path', () => { hostRef.current = null }) - it('serves the journal as terminal lines, with no dispatch and no capability', () => { + it('serves the journal as terminal lines, with no dispatch and no capability', async () => { // The defect this pins: a peer has no dispatch id and no coordinator standing, so `worker-read` // is closed to it, and `terminal read` threw `terminal_handle_stale` for a perfectly live // worker. A peer could not see a structured agent's recent output at all. const handle = registerWorker() installHost({ items: [message('i1', 'first line\nsecond line'), message('i2', 'done')] }) - const read = readStructuredWorkerTerminal({ handle, db: null }) + const read = await readStructuredWorkerTerminal({ handle, db: null }) expect(read?.tail).toEqual(['[assistant] first line', 'second line', '[assistant] done']) expect(read?.status).toBe('running') expect(read?.truncated).toBe(false) }) - it('honours limit, and claims no cursor space it cannot honour', () => { + it('honours limit, and claims no cursor space it cannot honour', async () => { const handle = registerWorker() installHost({ items: [message('i1', 'a'), message('i2', 'b'), message('i3', 'c')] }) - const read = readStructuredWorkerTerminal({ handle, db: null, limit: 2 }) + const read = await readStructuredWorkerTerminal({ handle, db: null, limit: 2 }) expect(read?.tail).toEqual(['[assistant] b', '[assistant] c']) // No index is advertised: the next read re-projects a sliding window, so 0/length would name // positions that address different lines by then. @@ -99,7 +99,7 @@ describe('reading a structured worker through the terminal-read path', () => { expect(read?.latestCursor).toBeUndefined() }) - it('refuses a cursor read rather than silently misdelivering lines', () => { + it('refuses a cursor read rather than silently misdelivering lines', async () => { // The PTY cursor indexes an append-only completed-line buffer with a monotone count. This // window is a bounded tail re-projected every read, so a saved index addresses different lines // as the journal grows — and `truncated` could never fire to say so, because it tests @@ -107,14 +107,10 @@ describe('reading a structured worker through the terminal-read path', () => { // duplicated lines with `truncated:false`. const handle = registerWorker() installHost({ items: [message('i1', 'a')] }) - const refusal = (() => { - try { - readStructuredWorkerTerminal({ handle, db: null, cursor: 0 }) - return '' - } catch (error) { - return (error as Error).message - } - })() + const refusal = await readStructuredWorkerTerminal({ handle, db: null, cursor: 0 }).then( + () => '', + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ) expect(refusal).toMatch(/not line-addressable/) // Tells the caller what DOES work here. Polling a bounded newest-last tail and diffing fails // safe — a harmless re-read — where a broken cursor fails unsafe, as a silent hole. @@ -125,35 +121,35 @@ describe('reading a structured worker through the terminal-read path', () => { expect(refusal).not.toContain('worker-read') }) - it('reports dropped history as truncated rather than pretending the page is whole', () => { + it('reports dropped history as truncated rather than pretending the page is whole', async () => { const handle = registerWorker() installHost({ items: [message('i1', 'tail only')], hasOlder: true }) - expect(readStructuredWorkerTerminal({ handle, db: null })?.truncated).toBe(true) + expect((await readStructuredWorkerTerminal({ handle, db: null }))?.truncated).toBe(true) }) - it('redacts dispatch capability tokens the same way the archive path does', () => { + it('redacts dispatch capability tokens the same way the archive path does', async () => { const handle = registerWorker() const token = `dcap_${'a'.repeat(32)}` installHost({ items: [message('i1', `token is ${token} here`)] }) - const tail = readStructuredWorkerTerminal({ handle, db: null })?.tail.join('\n') ?? '' + const tail = (await readStructuredWorkerTerminal({ handle, db: null }))?.tail.join('\n') ?? '' expect(tail).not.toContain(token) expect(tail).toContain('[dispatch capability redacted]') }) - it('refuses when the session is not attached rather than answering an empty tail', () => { + it('refuses when the session is not attached rather than answering an empty tail', async () => { // An empty tail is the claim "this worker has produced no output", which is a different and // false statement — and the one a caller cannot tell apart from a real silence. const handle = registerWorker() installHost({ items: 'unreadable' }) - expect(() => readStructuredWorkerTerminal({ handle, db: null })).toThrow( + await expect(readStructuredWorkerTerminal({ handle, db: null })).rejects.toThrow( 'agent_session_ownership_unknown' ) }) - it('reports a session it cannot verify as unknown, never as running', () => { + it('reports a session it cannot verify as unknown, never as running', async () => { const handle = registerWorker() installHost({ items: [message('i1', 'said something')], hasSession: false }) - expect(readStructuredWorkerTerminal({ handle, db: null })?.status).toBe('unknown') + expect((await readStructuredWorkerTerminal({ handle, db: null }))?.status).toBe('unknown') }) it('is what `terminal read` answers with, ahead of the PTY lookup', async () => { @@ -182,15 +178,15 @@ describe('reading a structured worker through the terminal-read path', () => { }) }) - it('leaves every handle that is not a live structured worker to the PTY path', () => { + it('leaves every handle that is not a live structured worker to the PTY path', async () => { const handle = registerWorker() installHost({ items: [message('i1', 'x')] }) - expect(readStructuredWorkerTerminal({ handle: 'term_abc', db: null })).toBeNull() + expect(await readStructuredWorkerTerminal({ handle: 'term_abc', db: null })).toBeNull() // A terminal owner an older build recorded loads conflicted: not this runtime's worker. installHost({ items: [message('i1', 'x')], lease: { runtimeKind: 'native', claimStatus: 'conflicted' } }) - expect(readStructuredWorkerTerminal({ handle, db: null })).toBeNull() + expect(await readStructuredWorkerTerminal({ handle, db: null })).toBeNull() }) }) diff --git a/src/main/runtime/structured-worker-terminal-read.ts b/src/main/runtime/structured-worker-terminal-read.ts index 9b4a118b8ae4..3d31c4c863c7 100644 --- a/src/main/runtime/structured-worker-terminal-read.ts +++ b/src/main/runtime/structured-worker-terminal-read.ts @@ -48,12 +48,12 @@ import { readTerminalTail } from './terminal-tail-read' * IS a structured worker never falls through: an unreadable journal refuses rather than answering * an empty tail, which a caller cannot tell from a worker that has said nothing. */ -export function readStructuredWorkerTerminal(args: { +export async function readStructuredWorkerTerminal(args: { handle: string db: OrchestrationDb | null cursor?: number limit?: number -}): RuntimeTerminalRead | null { +}): Promise { const identity = resolveStructuredWorkerAuthority(args.handle, args.db)?.identity if (!identity) { return null @@ -76,7 +76,7 @@ export function readStructuredWorkerTerminal(args: { 'A structured session has no durable line anchor to page from — nothing else does either.' ) } - const page = readStructuredJournalPage(identity.sessionId) + const page = await readStructuredJournalPage(identity.sessionId) if (!page) { // Honest refusal, and the same one the send lane reports: an empty tail would read as "this // worker has produced no output", which is a different and false claim. diff --git a/src/main/sqlite/sqlite-read-failure.test.ts b/src/main/sqlite/sqlite-read-failure.test.ts index 8bd43d54eaa4..553788063c21 100644 --- a/src/main/sqlite/sqlite-read-failure.test.ts +++ b/src/main/sqlite/sqlite-read-failure.test.ts @@ -1,9 +1,13 @@ -import { mkdtempSync, rmSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import SyncDatabase from './sync-database' -import { classifySqliteReadFailure, isTransientSqliteContention } from './sqlite-read-failure' +import { + classifySqliteReadFailure, + isTransientSqliteContention, + isUnusableSqliteDatabaseError +} from './sqlite-read-failure' // Error codes here are the ones a real node:sqlite open produces: a contended // database reports errcode 5 ("database is locked"), while a read-only WAL open @@ -111,3 +115,21 @@ describe('classifySqliteReadFailure', () => { ).toBe('unreadable') }) }) + +describe('isUnusableSqliteDatabaseError', () => { + it('recognizes the error a real open of a garbage file throws, and nothing transient', () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-sqlite-unusable-')) + tempDirs.push(dir) + const path = join(dir, 'garbage.db') + writeFileSync(path, Buffer.alloc(4096, 7)) + let thrown: unknown + try { + new SyncDatabase(path).pragma('user_version', { simple: true }) + } catch (error) { + thrown = error + } + expect(isUnusableSqliteDatabaseError(thrown)).toBe(true) + expect(isUnusableSqliteDatabaseError(new Error('database is locked'))).toBe(false) + expect(isUnusableSqliteDatabaseError({ errcode: 14, message: 'unable to open' })).toBe(false) + }) +}) diff --git a/src/main/sqlite/sqlite-read-failure.ts b/src/main/sqlite/sqlite-read-failure.ts index f96176eee0ac..f72ff3087965 100644 --- a/src/main/sqlite/sqlite-read-failure.ts +++ b/src/main/sqlite/sqlite-read-failure.ts @@ -6,7 +6,9 @@ /** Primary result codes; extended codes pack the primary code in the low byte. */ const SQLITE_BUSY = 5 const SQLITE_LOCKED = 6 +const SQLITE_CORRUPT = 11 const SQLITE_CANTOPEN = 14 +const SQLITE_NOTADB = 26 // Shared with the Codex index-heal pass, which only ever sees a relayed message // string (app-server RPC drops `errcode`), so message matching is not optional. @@ -24,6 +26,27 @@ function errorText(error: unknown): string { return error instanceof Error ? error.message : String(error) } +// SQLite reports a torn file at the first statement that reads a page, so this matches the message +// as well as the code. +const UNUSABLE_DATABASE = + /SQLITE_CORRUPT|SQLITE_NOTADB|file is not a database|database disk image is malformed/i + +/** True when the file is not a usable database at all, which no retry can change. */ +export function isUnusableSqliteDatabaseError(error: unknown): boolean { + const errcode = primaryErrcode(error) + if (errcode === SQLITE_CORRUPT || errcode === SQLITE_NOTADB) { + return true + } + if (!(error instanceof Error)) { + return false + } + const code = 'code' in error ? error.code : undefined + return ( + (typeof code === 'string' && UNUSABLE_DATABASE.test(code)) || + UNUSABLE_DATABASE.test(error.message) + ) +} + /** * True when a SQLite failure means "someone else holds the database right now". * @param error - The thrown value from a `node:sqlite` call or a relayed message. diff --git a/src/main/startup/main-process-ipc-bootstrap.test.ts b/src/main/startup/main-process-ipc-bootstrap.test.ts new file mode 100644 index 000000000000..18e5aa69b8a8 --- /dev/null +++ b/src/main/startup/main-process-ipc-bootstrap.test.ts @@ -0,0 +1,58 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { handlers, state } = vi.hoisted(() => { + const state: { + firstWindowStartupServicesReady: Promise + managedWslCliStartupBarrierReady: Promise + runtime: null | { prepareStructuredAgentSessionStartupRestoration: () => Promise } + } = { + firstWindowStartupServicesReady: Promise.resolve(), + managedWslCliStartupBarrierReady: Promise.resolve(), + runtime: null + } + return { handlers: new Map unknown>(), state } +}) + +vi.mock('electron', () => ({ + ipcMain: { + handle: (channel: string, handler: (...args: unknown[]) => unknown) => { + handlers.set(channel, handler) + } + } +})) +vi.mock('./main-process-state', () => ({ mainProcessState: state })) +vi.mock('./legacy-worker-renderer-recovery', () => ({ + recoverLegacyWorkerTerminalsForRendererStartup: vi.fn() +})) +vi.mock('./os-opened-markdown-files', () => ({ resolveOpenedMarkdownDocuments: vi.fn() })) + +import { registerMainProcessIpcHandlers } from './main-process-ipc-bootstrap' + +describe('app:prepareTerminalStartupRestoration', () => { + beforeEach(() => { + handlers.clear() + registerMainProcessIpcHandlers() + }) + + it('answers the renderer once the barriers settle while chat restoration still hangs', async () => { + const prepare = vi.fn(() => new Promise(() => {})) + state.runtime = { prepareStructuredAgentSessionStartupRestoration: prepare } + + await expect( + Promise.resolve(handlers.get('app:prepareTerminalStartupRestoration')?.()) + ).resolves.toBeUndefined() + expect(prepare).toHaveBeenCalledOnce() + }) + + it('reports a chat restoration failure without failing the renderer step', async () => { + const failure = new Error('host install failed') + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}) + state.runtime = { + prepareStructuredAgentSessionStartupRestoration: () => Promise.reject(failure) + } + + await expect(handlers.get('app:prepareTerminalStartupRestoration')?.()).resolves.toBeUndefined() + await vi.waitFor(() => expect(logged).toHaveBeenCalledWith(expect.any(String), failure)) + logged.mockRestore() + }) +}) diff --git a/src/main/startup/main-process-ipc-bootstrap.ts b/src/main/startup/main-process-ipc-bootstrap.ts index 918fd8443640..e9e0cf81db25 100644 --- a/src/main/startup/main-process-ipc-bootstrap.ts +++ b/src/main/startup/main-process-ipc-bootstrap.ts @@ -23,7 +23,11 @@ export function registerMainProcessIpcHandlers(): void { state.firstWindowStartupServicesReady, state.managedWslCliStartupBarrierReady ]) - await state.runtime?.prepareStructuredAgentSessionStartupRestoration() + // Not awaited: chat bookkeeping must not hold the window's first paint. Every hold, attach, + // send and worker release awaits the same reconcile itself; reads need none. + void state.runtime?.prepareStructuredAgentSessionStartupRestoration().catch((error) => { + console.error('[structured-agent-session] startup restoration failed', error) + }) }) ipcMain.handle('app:recoverLegacyWorkerTerminalsForRendererStartup', () => recoverLegacyWorkerTerminalsForRendererStartup({ diff --git a/src/renderer/src/app-shell/use-app-startup-hydration.ts b/src/renderer/src/app-shell/use-app-startup-hydration.ts index a2fd133ae32e..f150fc05901b 100644 --- a/src/renderer/src/app-shell/use-app-startup-hydration.ts +++ b/src/renderer/src/app-shell/use-app-startup-hydration.ts @@ -281,10 +281,12 @@ export function useAppStartupHydration(onOnboardingLoaded: (state: OnboardingSta await timeRendererStartupStep('recover-legacy-worker-terminals-post-reconnect', () => window.api.app.recoverLegacyWorkerTerminalsForRendererStartup() ) + // Not awaited: chat tabs come from the host's durable index and land whenever it answers. + // Until then the saved chat tabs stay as hydrated; the tab sync awaits this same restore. if (useAppStore.getState().settings?.experimentalStructuredNativeChat === true) { - await timeRendererStartupStep('project-structured-session-tabs', () => + void timeRendererStartupStep('project-structured-session-tabs', () => restoreLocalStructuredSessionTabsOnce() - ) + ).catch(() => {}) } if (cancelled) { return diff --git a/src/renderer/src/app-startup-routing.test.ts b/src/renderer/src/app-startup-routing.test.ts index fbaa404dd346..89819d1fd083 100644 --- a/src/renderer/src/app-startup-routing.test.ts +++ b/src/renderer/src/app-startup-routing.test.ts @@ -376,6 +376,16 @@ describe('renderer startup runtime routing', () => { ) }) + it('never holds the session writer on the structured tab projection (L13)', () => { + const source = readSource(STARTUP_HYDRATION_PATH) + const projectIndex = source.indexOf("timeRendererStartupStep('project-structured-session-tabs'") + const unlockIndex = source.indexOf('actions.setHydrationSucceeded(true)') + + expect(projectIndex).toBeGreaterThanOrEqual(0) + expect(source.slice(projectIndex - 'void '.length, projectIndex)).toBe('void ') + expect(source.slice(projectIndex, unlockIndex)).not.toContain('await ') + }) + it('probes local runtime capabilities before any startup gate can hold the answer back', () => { const source = readSource(STARTUP_HYDRATION_PATH) const probeIndex = source.indexOf('void ensureLocalRuntimeCapabilities()') diff --git a/src/renderer/src/components/native-chat-resume-failed-chat-watch.test.ts b/src/renderer/src/components/native-chat-resume-failed-chat-watch.test.ts index 308e7b878e89..7dfc8938373e 100644 --- a/src/renderer/src/components/native-chat-resume-failed-chat-watch.test.ts +++ b/src/renderer/src/components/native-chat-resume-failed-chat-watch.test.ts @@ -25,8 +25,8 @@ vi.mock('@/runtime/structured-agent-session-client', () => ({ })) vi.mock('sonner', () => ({ toast: vi.fn() })) -// After a restart action leaves a chat failed, the user's reply in that chat must retire the status -// bar entry without the user reopening anything, and nothing may run while nothing failed. +// When the user sends in an offered or failed chat, its status bar entry must retire without the +// user reopening anything, and nothing may run while nothing is offered or failed. const failure = { sessionId: 'a', @@ -91,7 +91,26 @@ afterEach(() => { vi.useRealTimers() }) -it('opens no status stream while nothing failed', async () => { +// The host withdraws a pending offer once the chat moves on; the status bar must not keep counting it. +it('re-reads once when the user sends in an offered chat, and the count drops', async () => { + const { failedAt: _failedAt, outcome: _outcome, reason: _reason, ...candidate } = failure + mocks.rpc + .mockResolvedValueOnce({ sessions: [candidate], failed: [] }) + .mockResolvedValue({ sessions: [], failed: [] }) + await refreshNativeChatRestartOffer() + await vi.advanceTimersByTimeAsync(0) + expect(getNativeChatRestartOffer().candidates).toHaveLength(1) + expect(mocks.subscribeStatus).toHaveBeenCalledOnce() + + hostEmit()({ type: 'status', session: summary('working', 'Something else', Date.now() + 10_000) }) + await vi.advanceTimersByTimeAsync(500) + + expect(offerReads()).toBe(2) + expect(getNativeChatRestartOffer().candidates).toEqual([]) + expect(mocks.unsubscribe).toHaveBeenCalledOnce() +}) + +it('opens no status stream while nothing is offered or failed', async () => { mocks.rpc.mockResolvedValue({ sessions: [], failed: [] }) await refreshNativeChatRestartOffer() await vi.advanceTimersByTimeAsync(0) diff --git a/src/renderer/src/components/native-chat-resume-on-restart-store.ts b/src/renderer/src/components/native-chat-resume-on-restart-store.ts index a3012495a3c4..78c5fa207b67 100644 --- a/src/renderer/src/components/native-chat-resume-on-restart-store.ts +++ b/src/renderer/src/components/native-chat-resume-on-restart-store.ts @@ -58,42 +58,47 @@ const LAUNCH_READ_RETRY_DELAYS_MS = [100, 250, 500] as const * actually moves the offer. */ function publish(next: NativeChatRestartOffer): void { offer = next - syncFailedChatWatch() + syncOfferedChatWatch() for (const listener of listeners) { listener() } } /** - * Re-reads the host once a failed chat shows new activity, so a reply the user sent there retires - * its entry here too. The host stays the judge; this only asks again. + * Re-reads the host once an offered or failed chat shows new activity, so a message the user sent + * there, or its agent starting, retires its entry here too. The host stays the judge; this only + * asks again. * - * Held only while something failed. Keyed on status and prompt rather than every summary, so an - * agent streaming in a failed chat costs one re-read, not one per tool call. + * Held only while something is offered or failed. Keyed on status and prompt rather than every + * summary, so an agent streaming in such a chat costs one re-read, not one per tool call. */ -const FAILED_CHAT_REFRESH_DELAY_MS = 500 -let failedChatWatch: { +const OFFERED_CHAT_REFRESH_DELAY_MS = 500 +let offeredChatWatch: { feed: StructuredAgentSessionStatusFeedOwner seen: Map release: () => void } | null = null -let failedChatRefresh: ReturnType | null = null +let offeredChatRefresh: ReturnType | null = null -function failedChatActivityKey(summary: AgentSessionStatusSummary): string { +function offeredChatIds(): Set { + return new Set([...offer.candidates, ...offer.failed].map((entry) => entry.sessionId)) +} + +function offeredChatActivityKey(summary: AgentSessionStatusSummary): string { return `${summary.status ?? ''}\u0000${summary.latestPrompt}` } -function syncFailedChatWatch(): void { - const failedIds = new Set(offer.failed.map((failure) => failure.sessionId)) - if (failedIds.size === 0) { - releaseFailedChatWatch() +function syncOfferedChatWatch(): void { + const offeredIds = offeredChatIds() + if (offeredIds.size === 0) { + releaseOfferedChatWatch() return } - if (!failedChatWatch) { + if (!offeredChatWatch) { const feed = getStructuredAgentSessionStatusFeed(LOCAL) - const unsubscribe = feed.subscribe(noticeFailedChatActivity) + const unsubscribe = feed.subscribe(noticeOfferedChatActivity) const deactivate = feed.activate() - failedChatWatch = { + offeredChatWatch = { feed, seen: new Map(), release: () => { @@ -102,60 +107,60 @@ function syncFailedChatWatch(): void { } } } - const { feed, seen } = failedChatWatch + const { feed, seen } = offeredChatWatch for (const sessionId of seen.keys()) { - if (!failedIds.has(sessionId)) { + if (!offeredIds.has(sessionId)) { seen.delete(sessionId) } } // What the feed already holds is what this listing answered. - for (const sessionId of failedIds) { + for (const sessionId of offeredIds) { const summary = feed.getSnapshot().get(sessionId) if (summary && !seen.has(sessionId)) { - seen.set(sessionId, failedChatActivityKey(summary)) + seen.set(sessionId, offeredChatActivityKey(summary)) } } } -function noticeFailedChatActivity(): void { - if (!failedChatWatch) { +function noticeOfferedChatActivity(): void { + if (!offeredChatWatch) { return } - const { feed, seen } = failedChatWatch + const { feed, seen } = offeredChatWatch const snapshot = feed.getSnapshot() let changed = false - for (const failure of offer.failed) { - const summary = snapshot.get(failure.sessionId) + for (const sessionId of offeredChatIds()) { + const summary = snapshot.get(sessionId) if (!summary) { continue } - const key = failedChatActivityKey(summary) - const previous = seen.get(failure.sessionId) + const key = offeredChatActivityKey(summary) + const previous = seen.get(sessionId) if (previous !== key) { - seen.set(failure.sessionId, key) + seen.set(sessionId, key) // A first sighting is news only if newer than the list; a change to a known chat always is, // since the host may have answered the list just before the change was delivered here. changed ||= previous !== undefined || summary.updatedAt > offer.listedAt } } - if (changed && failedChatRefresh === null) { - failedChatRefresh = setTimeout(() => { - failedChatRefresh = null + if (changed && offeredChatRefresh === null) { + offeredChatRefresh = setTimeout(() => { + offeredChatRefresh = null if (actionsBegun === actionsSettled) { const issued = actionsBegun void readNativeChatRestartOffer(() => actionsBegun === issued) } - }, FAILED_CHAT_REFRESH_DELAY_MS) + }, OFFERED_CHAT_REFRESH_DELAY_MS) } } -function releaseFailedChatWatch(): void { - if (failedChatRefresh !== null) { - clearTimeout(failedChatRefresh) - failedChatRefresh = null +function releaseOfferedChatWatch(): void { + if (offeredChatRefresh !== null) { + clearTimeout(offeredChatRefresh) + offeredChatRefresh = null } - failedChatWatch?.release() - failedChatWatch = null + offeredChatWatch?.release() + offeredChatWatch = null } export function getNativeChatRestartOffer(): NativeChatRestartOffer { @@ -358,7 +363,7 @@ export function useNativeChatRestartOffer(enabled: boolean): NativeChatRestartOf /** @internal - tests need a clean module between cases. */ export function _resetNativeChatRestartOffer(): void { - releaseFailedChatWatch() + releaseOfferedChatWatch() offer = EMPTY actionsBegun = 0 actionsSettled = 0 diff --git a/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx b/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx index 5958e2a8d543..74da4709e938 100644 --- a/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx +++ b/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx @@ -16,9 +16,13 @@ export function NativeChatDeliveryRetry({ retry: (clientMessageId: string) => void }): React.JSX.Element | null { // Why: read through the drain's own rule, so Retry can never name an entry other than the one - // the queue actually stopped on -- which is no longer always the head. + // the queue actually stopped on -- which is no longer always the head. A rejected entry holds + // nothing up, so it is offered only when the queue itself is not stopped. const admission = admitStructuredAgentSessionOutboxEntry(outbox, blockedClientMessageId) - const retryable = admission.state === 'blocked' ? admission.entry : null + const retryable = + admission.state === 'blocked' + ? admission.entry + : (outbox.find((entry) => entry.state === 'rejected') ?? null) if (!retryable) { return null } diff --git a/src/renderer/src/components/native-chat/NativeChatEmptyState.test.tsx b/src/renderer/src/components/native-chat/NativeChatEmptyState.test.tsx new file mode 100644 index 000000000000..3fa684b0a519 --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatEmptyState.test.tsx @@ -0,0 +1,30 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' + +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, expect, it } from 'vitest' +import { NativeChatEmptyState } from './NativeChatEmptyState' + +afterEach(cleanup) + +it('keeps pointing the terminal-backed chat back to its terminal when a read fails', () => { + render() + expect( + screen.getByText( + 'The transcript could not be read. Toggle back to the terminal to keep working.' + ) + ).toBeInTheDocument() +}) + +it('tells the structured chat its read keeps retrying', () => { + render() + expect( + screen.getByText('The transcript could not be read. Orca keeps trying to load it.') + ).toBeInTheDocument() +}) + +it('shows the host message over either default', () => { + render() + expect(screen.getByText('disk full')).toBeInTheDocument() +}) diff --git a/src/renderer/src/components/native-chat/NativeChatEmptyState.tsx b/src/renderer/src/components/native-chat/NativeChatEmptyState.tsx index f7c20ef51f45..eb0a9e995d96 100644 --- a/src/renderer/src/components/native-chat/NativeChatEmptyState.tsx +++ b/src/renderer/src/components/native-chat/NativeChatEmptyState.tsx @@ -7,13 +7,17 @@ import type { NativeChatSession } from '../../../../shared/native-chat-types' export function NativeChatEmptyState({ kind, message, - agent + agent, + retrying = false }: { kind: 'loading' | 'empty' | 'error' | 'not-agent' message?: string agent?: NativeChatSession['agent'] + /** The read retries on its own (structured chat), so the error says so instead of pointing + * back to the terminal. */ + retrying?: boolean }): React.JSX.Element { - const copy = emptyStateCopy(kind, message, agent) + const copy = emptyStateCopy(kind, message, agent, retrying) return (
) : viewState.kind === 'error' ? ( - + ) : viewState.kind === 'empty' ? ( ) : ( diff --git a/src/renderer/src/components/native-chat/native-chat-view-state.test.ts b/src/renderer/src/components/native-chat/native-chat-view-state.test.ts index ea97ec99a191..39bf20322b50 100644 --- a/src/renderer/src/components/native-chat/native-chat-view-state.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-view-state.test.ts @@ -34,7 +34,30 @@ describe('selectNativeChatViewState', () => { }) }) - it('maps error with its message', () => { + it('maps error with its message when nothing has loaded (P2-21)', () => { + const state = selectNativeChatViewState( + session({ messages: [], status: 'error', error: 'boom' }) + ) + expect(state).toEqual({ kind: 'error', message: 'boom' }) + }) + + // The pane's own translated line fills in, which for the structured chat says the read retries. + it('maps an error with no text to no message', () => { + expect(selectNativeChatViewState(session({ messages: [], status: 'error' }))).toEqual({ + kind: 'error' + }) + }) + + it('keeps a loaded transcript on screen when a later read fails (P2-21)', () => { + const state = selectNativeChatViewState(session({ status: 'error', error: 'boom' }), { + readRetries: true + }) + expect(state).toEqual({ kind: 'ready', isWorking: false }) + }) + + // The terminal-backed read does not retry; its only messages on error are local echoes (a launch + // prompt, a pending send), which must not hide the error and its way back to the terminal. + it('shows a terminal-backed read error over local echoes', () => { const state = selectNativeChatViewState(session({ status: 'error', error: 'boom' })) expect(state).toEqual({ kind: 'error', message: 'boom' }) }) diff --git a/src/renderer/src/components/native-chat/native-chat-view-state.ts b/src/renderer/src/components/native-chat/native-chat-view-state.ts index 775635df1229..13a7e8a5afe6 100644 --- a/src/renderer/src/components/native-chat/native-chat-view-state.ts +++ b/src/renderer/src/components/native-chat/native-chat-view-state.ts @@ -10,23 +10,36 @@ import type { NativeChatSession } from '../../../../shared/native-chat-types' * live in-flight indicator. The rest are full-pane states. */ export type NativeChatViewState = | { kind: 'loading' } - | { kind: 'error'; message: string } + | { kind: 'error'; message?: string } | { kind: 'empty' } | { kind: 'ready'; isWorking: false } | { kind: 'ready'; isWorking: true } /** - * Decide which surface to render. Error is terminal, but any renderable message - * wins over loading/empty so optimistic first sends never get replaced by a - * full-pane placeholder while transcript discovery catches up. + * Decide which surface to render. Any renderable message wins over loading/empty so optimistic + * first sends never get replaced while transcript discovery catches up. Where the read retries on + * its own (the structured chat), messages win over an error too: a read that fails after the + * transcript loaded keeps it, and the error reaches the composer's error line. The terminal-backed + * read does not retry, and its only messages on error are local echoes, so its error takes the pane. */ -export function selectNativeChatViewState(session: NativeChatSession): NativeChatViewState { - if (session.status === 'error') { - return { kind: 'error', message: session.error ?? 'Conversation could not be loaded.' } +export function selectNativeChatViewState( + session: NativeChatSession, + { readRetries = false }: { readRetries?: boolean } = {} +): NativeChatViewState { + // No text of its own: the empty state supplies the pane's translated line. + const error: NativeChatViewState | null = + session.status === 'error' + ? { kind: 'error', ...(session.error ? { message: session.error } : {}) } + : null + if (error && !readRetries) { + return error } if (session.messages.length > 0) { return { kind: 'ready', isWorking: session.status === 'working' } } + if (error) { + return error + } if (session.status === 'loading') { return { kind: 'loading' } } diff --git a/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts b/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts index fc2d5adf1508..e0378b6cfe34 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts @@ -56,6 +56,18 @@ export function readMountedStructuredAgentSessionOutbox( ) } +/** A send left dispatching when its owner changed goes out again, under the same id. */ +export function requeueInterruptedStructuredAgentSessionDispatches( + entries: StructuredAgentSessionOutboxEntry[], + fence: number | null +): StructuredAgentSessionOutboxEntry[] { + return entries.map((entry) => + entry.state === 'dispatching' && !hasInFlightLaunchDispatch(entry, fence) + ? { ...entry, state: 'queued' as const } + : entry + ) +} + export function dispatchStructuredAgentSessionOutboxEntry(args: { next: StructuredAgentSessionOutboxEntry persisted: readonly StructuredAgentSessionOutboxEntry[] diff --git a/src/renderer/src/components/native-chat/structured-agent-session-read-transport.test.ts b/src/renderer/src/components/native-chat/structured-agent-session-read-transport.test.ts index 5e5be1b6b3b4..29b0a80b46be 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-read-transport.test.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-read-transport.test.ts @@ -269,7 +269,7 @@ describe('structured agent-session read transport unattached refusals', () => { // ...and the failure is owed once it has passed. await vi.advanceTimersByTimeAsync(750) attempts.at(-1)?.onError(rpcRefusal(UNATTACHED)) - expect(applyError).toHaveBeenCalledWith(`RuntimeRpcCallError: ${UNATTACHED}`) + expect(applyError).toHaveBeenCalledWith(UNATTACHED) transport.dispose() } finally { vi.useRealTimers() @@ -284,7 +284,7 @@ describe('structured agent-session read transport unattached refusals', () => { throw new Error('journal read failed') }, applyError) await flushPromises() - expect(applyError).toHaveBeenCalledExactlyOnceWith('Error: journal read failed') + expect(applyError).toHaveBeenCalledExactlyOnceWith('journal read failed') transport.dispose() } finally { vi.useRealTimers() @@ -303,7 +303,51 @@ describe('structured agent-session read transport unattached refusals', () => { expect(applyError).not.toHaveBeenCalled() attempts[0].onError({ code: 'runtime_error', message: 'transport died' }) - expect(applyError).toHaveBeenCalledOnce() + // The host's own words, never `[object Object]` (P2-04). + expect(applyError).toHaveBeenCalledExactlyOnceWith('transport died') + transport.dispose() + } finally { + vi.useRealTimers() + } + }) + + it('re-opens after a failed open and leaves the error once the conversation reads (P2-04)', async () => { + vi.useFakeTimers() + try { + const applyError = vi.fn() + const applyEvent = vi.fn() + const transport = startWithHydration(async () => undefined, applyError, applyEvent) + await flushPromises() + attempts[0].onError({ code: 'agent_session_journal_unreadable', message: 'disk full' }) + attempts[0].closed.resolve({ unsubscribe: attempts[0].unsubscribe }) + await flushPromises() + expect(applyError).toHaveBeenCalledExactlyOnceWith('disk full') + + // The retry re-asks, and the host's open runs again: a fault that cleared now reads. + await vi.advanceTimersByTimeAsync(750) + expect(attempts).toHaveLength(2) + attempts[1].onEvent(snapshot(1)) + await flushPromises() + expect(applyEvent).toHaveBeenCalledWith(expect.objectContaining({ type: 'snapshot' })) + transport.dispose() + } finally { + vi.useRealTimers() + } + }) + + it('stops re-asking a history whose journal file is not a usable database', async () => { + vi.useFakeTimers() + try { + const applyError = vi.fn() + const transport = startWithHydration(async () => undefined, applyError) + await flushPromises() + // The host raises it with the bare code as the message; the RPC layer's code is generic. + attempts[0].onError({ code: 'runtime_error', message: 'agent_session_journal_unreadable' }) + attempts[0].closed.resolve({ unsubscribe: attempts[0].unsubscribe }) + await flushPromises() + expect(applyError).toHaveBeenCalledExactlyOnceWith('agent_session_journal_unreadable') + await vi.advanceTimersByTimeAsync(10_000) + expect(attempts).toHaveLength(1) transport.dispose() } finally { vi.useRealTimers() diff --git a/src/renderer/src/components/native-chat/structured-agent-session-read-transport.ts b/src/renderer/src/components/native-chat/structured-agent-session-read-transport.ts index f4310cec8904..64d29dad7c01 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-read-transport.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-read-transport.ts @@ -3,8 +3,10 @@ import type { AgentSessionSubscribeEvent } from '../../../../shared/agent-sessio import { createStructuredAgentSessionEventCoalescer } from '../../../../shared/structured-agent-session-coalescer' import { AGENT_SESSION_UNATTACHED_READ_GRACE_MS, + isFinalAgentSessionReadRefusal, isUnattachedAgentSessionReadRefusal } from '../../../../shared/structured-agent-session-read-refusal' +import { agentSessionErrorText } from '../../../../shared/agent-session-error-text' import type { RuntimeClientTarget } from '@/runtime/runtime-rpc-client' import { subscribeStructuredAgentSession } from '@/runtime/structured-agent-session-client' @@ -44,6 +46,8 @@ export function startStructuredAgentSessionReadTransport(args: { dispose: () => void } { let stopped = false + // An unusable journal file answers the same on every ask, so nothing re-asks it. + let finalRefusal = false let connected = false let unattachedSince: number | null = null let opening = false @@ -57,7 +61,7 @@ export function startStructuredAgentSessionReadTransport(args: { } }) const reconnectScheduler = createReconnectScheduler({ - shouldStop: () => stopped || connected, + shouldStop: () => stopped || connected || finalRefusal, reconnect: () => void open() }) const isCurrentOpenGeneration = (candidate: number): boolean => @@ -69,23 +73,24 @@ export function startStructuredAgentSessionReadTransport(args: { * A read failure, reported to the pane only once it is one. * * A refusal saying this host holds no attached session is the retry loop's own subject, not a - * verdict: the reconnect below re-asks, and either the hold that attaches the session or the tab - * retirement that ends the pane resolves it. Painting the pane red inside that window turned an + * verdict: the reconnect below re-asks, and either the host opening the session or the tab + * retirement that ends the pane resolves it. Only an older host still refuses this way. Painting the pane red inside that window turned an * ordinary chat close into a `Could not load conversation` the user could do nothing about. * * A window, not a mute. An unattached read still refusing past the grace is no longer * transitional, so the pane is owed the failure rather than a spinner that never resolves. */ const reportReadFailure = (error: unknown): void => { + finalRefusal = isFinalAgentSessionReadRefusal(error) if (!isUnattachedAgentSessionReadRefusal(error)) { clearUnattachedReadGrace() - args.applyError(String(error)) + args.applyError(agentSessionErrorText(error)) return } const now = Date.now() unattachedSince ??= now if (now - unattachedSince >= AGENT_SESSION_UNATTACHED_READ_GRACE_MS) { - args.applyError(String(error)) + args.applyError(agentSessionErrorText(error)) } } const captureHistoryReadGuard = (): (() => boolean) => { diff --git a/src/renderer/src/components/native-chat/use-native-chat-structured-composer-send.ts b/src/renderer/src/components/native-chat/use-native-chat-structured-composer-send.ts index f208e6cb4a93..2db9fd248901 100644 --- a/src/renderer/src/components/native-chat/use-native-chat-structured-composer-send.ts +++ b/src/renderer/src/components/native-chat/use-native-chat-structured-composer-send.ts @@ -5,6 +5,7 @@ import { isStructuredAgentSessionComposerCommand, isStructuredAgentSessionGoalCommand } from '../../../../shared/structured-agent-session-composer' +import { agentSessionErrorText } from '../../../../shared/agent-session-error-text' import type { AgentType } from '../../../../shared/agent-status-types' import { dispatchNativeChatStructuredComposerText } from './native-chat-structured-composer-dispatch' import { pushHistory, type HistoryState } from './native-chat-composer-state' @@ -83,9 +84,7 @@ export function useNativeChatStructuredComposerSend({ clearSkillOrigin() clearImageAttachments() }) - .catch((error) => - structuredTransport.onError(error instanceof Error ? error.message : String(error)) - ) + .catch((error) => structuredTransport.onError(agentSessionErrorText(error))) }, [ agent, diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-hold.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-hold.ts index 628a369281bb..f484f500a56f 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-hold.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-hold.ts @@ -8,6 +8,9 @@ // // The release CHAINS off the hold rather than racing it: an unmount during the hold's round trip // would otherwise release a hold that has not landed yet, and the late hold would never be undone. +// +// A current host ignores both: it starts an agent only for a send. They stay for older hosts, until +// MIN_COMPATIBLE_RUNTIME_SERVER_VERSION passes the first host that ignores them. import { useEffect, useRef } from 'react' import { structuredAgentSessionHolderId } from '../../../../shared/structured-agent-session-holder' diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts index d7706913d21d..1ee70c572235 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts @@ -7,6 +7,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import * as conversationCommands from './structured-conversation-command-send' +import { agentSessionErrorText } from '../../../../shared/agent-session-error-text' import type { AgentSessionMutationResult } from '../../../../shared/agent-session-wire' import { agentSessionRefusalOperationState } from '../../../../shared/agent-session-refusal-retry' import { structuredAgentSessionPayloadFingerprint } from '../../../../shared/structured-agent-session-mutation' @@ -75,7 +76,7 @@ export function useStructuredAgentSessionMutate(args: { }) } catch (error) { if (enabledRef.current && stateRef.current.fence === targetFence) { - setWriteError(error instanceof Error ? error.message : 'Request was not sent') + setWriteError(agentSessionErrorText(error, 'Request was not sent')) } return null } diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx new file mode 100644 index 000000000000..fd306a537307 --- /dev/null +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx @@ -0,0 +1,138 @@ +// @vitest-environment happy-dom + +// A moved fence is not a reason to send anything again on a host that records every send before it +// starts an agent. There, only a Retry or a new send goes out. An older host, which restarts the +// agent inside the send and refuses it unrecorded when that fails, keeps the resend on a new fence. + +import { act, renderHook, waitFor } from '@testing-library/react' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' + +const mocks = vi.hoisted(() => ({ call: vi.fn() })) + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: mocks.call +})) + +import { setLocalRuntimeCapabilitiesForTests } from '@/runtime/local-runtime-capabilities' +import { useStructuredAgentSessionOutbox } from './use-structured-agent-session-outbox' + +const LOCAL_TARGET = { kind: 'local' } as const + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} + +function pendingResult(clientMessageId: string) { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 1 }, + value: { + clientMessageId, + submission: { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending' as const, + handoverRecorded: true, + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null + } + } + } +} + +function sentId(call: number): string { + const id: unknown = mocks.call.mock.calls[call]?.[2].envelope.clientOperationId + return String(id) +} + +function render() { + return renderHook( + ({ fence }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target: LOCAL_TARGET, + fence, + submissions: [] + }), + { initialProps: { fence: 1 } } + ) +} + +/** Long enough for any effect a fence change schedules to have sent. */ +async function settle(): Promise { + await act(() => new Promise((resolve) => setTimeout(resolve, 50))) +} + +describe('an outbox on a host that accepts a send before any agent has it', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + setLocalRuntimeCapabilitiesForTests([AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY]) + }) + + it('neither resends nor drops the answer of a send in flight when the fence moves', async () => { + const answer = deferred>() + mocks.call.mockReturnValueOnce(answer.promise) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(1)) + // A start or restart on the host moves the fence while the send is out. + rerender({ fence: 2 }) + rerender({ fence: 3 }) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + + await act(async () => answer.resolve(pendingResult(sentId(0)))) + // The answer lands: the entry is the host's now, not re-queued behind a moved fence. + expect(result.current.outbox).toMatchObject([{ state: 'dispatching' }]) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + }) + + it('keeps a blocked head blocked across a fence change; only Retry sends it', async () => { + mocks.call.mockRejectedValueOnce(new Error('send failed')).mockResolvedValue({ ok: true }) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.blockedClientMessageId).not.toBeNull()) + rerender({ fence: 2 }) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + expect(result.current.error).toBe('Error: send failed') + + act(() => result.current.retry(result.current.outbox[0]!.clientMessageId)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + }) +}) + +describe('an outbox on an older host', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + setLocalRuntimeCapabilitiesForTests([]) + }) + + it('still resends a send in flight when the fence moves, as the new owner may take it', async () => { + mocks.call.mockReturnValueOnce(new Promise(() => {})).mockReturnValue(new Promise(() => {})) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(1)) + rerender({ fence: 2 }) + + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + expect(sentId(1)).toBe(sentId(0)) + }) +}) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx index d66f579ed0d8..fdbb5c67b06a 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx @@ -2,6 +2,8 @@ import { act, renderHook, waitFor } from '@testing-library/react' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' +import { DISPATCH_REJECTED_CANCELLED } from '../../../../shared/structured-agent-session-dispatch-rejection' const mocks = vi.hoisted(() => ({ call: vi.fn() @@ -17,6 +19,28 @@ import { useStructuredAgentSessionOutbox } from './use-structured-agent-session- const REASON = 'The provider stopped before it finished starting: claude stream-json exited (code 1): claude: not signed in.' +function acceptedResultFor(clientMessageId: string) { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { + clientMessageId, + submission: { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'accepted', + providerItemId: `provider-${clientMessageId}`, + reason: null, + submittedAt: 1, + resolvedAt: 1 + } + } + } +} + function rejectedResultFor(clientMessageId: string) { return { ok: true, @@ -65,7 +89,231 @@ describe('a send the host rejected because the agent never started', () => { act(() => expect(result.current.send('hello')).toBe(true)) await waitFor(() => expect(result.current.error).toBe(REASON)) - expect(result.current.outbox[0]?.state).toBe('queued') - expect(result.current.blockedClientMessageId).toBe(result.current.outbox[0]?.clientMessageId) + // Settled as not delivered: it waits for Retry and holds no later message up. + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() + }) + + it('sends a new message past one the host could not start the agent for, without resending it', async () => { + const message = "Claude couldn't restart: Not logged in. Please run /login." + mocks.call.mockImplementation(async (_target, _method, params) => { + const request = params as { + envelope: { clientOperationId: string } + body: { blocks: { text?: string }[] } + } + return request.body.blocks[0]?.text === 'first' + ? { ok: false, refusal: { code: 'agent_session_owner_restart_failed', message } } + : acceptedResultFor(request.envelope.clientOperationId) + }) + const { result } = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target: { kind: 'local' }, + fence: 1, + submissions: [] + }) + ) + + act(() => expect(result.current.send('first')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + const rejectedId = result.current.outbox[0]!.clientMessageId + + // The user's next message is the retry of the start: it goes out on its own. + act(() => expect(result.current.send('second')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + await act(() => new Promise((resolve) => setTimeout(resolve, 50))) + + const sent = mocks.call.mock.calls.map( + (call) => (call[2] as { body?: { blocks?: { text?: string }[] } })?.body?.blocks?.[0]?.text + ) + expect(sent).toEqual(['first', 'second']) + expect(result.current.outbox.map((entry) => [entry.clientMessageId, entry.state])).toEqual([ + [rejectedId, 'rejected'] + ]) + }) + + it('keeps a message the host accepted and then could not deliver, with its reason and Retry', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + rerender({ + submissions: [{ ...pendingResultFor(id).value.submission, dispatchState: 'rejected', reason }] + }) + + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + expect(result.current.error).toBe(reason) + expect(result.current.blockedClientMessageId).toBeNull() + + // Retry is a new message with the same text: a fresh id, sent once. + act(() => result.current.retry(id)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + const retried: { + envelope: { clientOperationId: string } + body: { blocks: { text?: string }[] } + } = mocks.call.mock.calls[1]![2] + expect(retried.envelope.clientOperationId).not.toBe(id) + expect(retried.body.blocks[0]?.text).toBe('hello') + }) + + it('says nothing when a Stop withdrew the message', async () => { + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + rerender({ + submissions: [ + { + ...pendingResultFor(id).value.submission, + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + } + ] + }) + + await waitFor(() => expect(result.current.outbox).toEqual([])) + expect(result.current.error).toBeNull() + }) + + it('reads a message rejected while the chat was closed as not sent, and sends past it', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const first = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: NO_SUBMISSIONS + }) + ) + act(() => expect(first.result.current.send('hello')).toBe(true)) + await waitFor(() => expect(first.result.current.outbox[0]?.state).toBe('dispatching')) + const id = first.result.current.outbox[0]!.clientMessageId + first.unmount() + + // Reopened after the start failed, or after a quit settled the message as not sent. + const rejected = [ + { ...pendingResultFor(id).value.submission, dispatchState: 'rejected' as const, reason } + ] + const { result } = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: rejected + }) + ) + + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + act(() => expect(result.current.send('second')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + }) + + it('keeps the rejection when the journal settles the message before the send answers', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + let answer: (value: unknown) => void = () => undefined + mocks.call.mockImplementationOnce( + () => + new Promise((resolve) => { + answer = resolve + }) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + // A start refused at once: the rejection frame lands before the send's own `pending` answer. + rerender({ + submissions: [{ ...pendingResultFor(id).value.submission, dispatchState: 'rejected', reason }] + }) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + await act(async () => answer(pendingResultFor(id))) + + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.error).toBe(reason) }) }) + +const NO_SUBMISSIONS: AgentJournalSubmission[] = [] + +function pendingResultFor(clientMessageId: string) { + const submission: AgentJournalSubmission = { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending', + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true + } + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { + clientMessageId, + submission + } + } +} diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx index 662ed752582b..d661b5f448ab 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx @@ -248,7 +248,7 @@ describe('useStructuredAgentSessionOutbox', () => { ) act(() => expect(result.current.send('hello')).toBe(true)) - await waitFor(() => expect(result.current.outbox[0]?.state).toBe('queued')) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) const firstId = (mocks.call.mock.calls[0]![2] as { envelope: { clientOperationId: string } }) .envelope.clientOperationId const retryId = result.current.outbox[0]!.clientMessageId @@ -551,7 +551,9 @@ describe('useStructuredAgentSessionOutbox', () => { expect(mocks.call).toHaveBeenCalledOnce() expect(result.current.outbox).toHaveLength(1) - expect(result.current.blockedClientMessageId).toBe(result.current.outbox[0]?.clientMessageId) + // Never sent, and never re-sent on its own: it waits for Retry and holds nothing up. + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() // Settled, not pending: the refused id never ran, so a Retry is a new operation. const sentId: unknown = mocks.call.mock.calls[0]![2].envelope.clientOperationId const retryId = result.current.outbox[0]!.clientMessageId @@ -773,8 +775,8 @@ describe('useStructuredAgentSessionOutbox', () => { "Couldn't reach the agent. Your message was not sent — Retry to send it again." ) ) - expect(result.current.outbox[0]?.state).toBe('queued') - expect(result.current.blockedClientMessageId).toBe(firstId) + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() // Retry immediately, before the journal subscription can publish the rejected row. act(() => result.current.retry(firstId)) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts index 17a9e2769f3f..e8fa73502042 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts @@ -7,15 +7,20 @@ import { reconcileStructuredAgentSessionOutbox, type StructuredAgentSessionOutboxEntry } from '../../../../shared/structured-agent-session-outbox' -import type { StructuredAgentSessionSendDisposition } from '../../../../shared/structured-agent-session-send-disposition' +import { + journalAnswersInFlightSend, + reconciledRejectionNotice, + type StructuredAgentSessionSendDisposition +} from '../../../../shared/structured-agent-session-send-disposition' import type { RuntimeClientTarget } from '@/runtime/runtime-rpc-client' import { readOutbox, writeOutbox } from './structured-agent-session-outbox-storage' import { dispatchStructuredAgentSessionOutboxEntry, - hasInFlightLaunchDispatch, - readMountedStructuredAgentSessionOutbox + readMountedStructuredAgentSessionOutbox, + requeueInterruptedStructuredAgentSessionDispatches } from './structured-agent-session-outbox-dispatch' import { getStructuredAgentLaunchPromptDispatch } from '@/lib/structured-agent-session-launch-prompt' +import { useStructuredAgentSessionOutboxOwnerChange } from '@/runtime/structured-agent-session-accepted-send-capability' import { createBrowserUuid } from '@/lib/browser-uuid' export function structuredSessionOperationId(): string { @@ -25,8 +30,8 @@ export function structuredSessionOperationId(): string { const UNCONFIRMED_PROBE_BASE_DELAY_MS = 1_000 /** No attempt ceiling: a transport outage outlives any fixed budget, and giving up * restores the wedge this fixes. Growth caps the rate at one status query per 16s. - * A refusal that blocks the head still ends probing until a fence change or a manual - * Retry, because the entry leaves `unconfirmed` -- pre-existing, not closed here. */ + * A refusal that blocks the head still ends probing until a manual Retry (or, on an older + * host, a fence change), because the entry leaves `unconfirmed`. */ const UNCONFIRMED_PROBE_MAX_DELAY_MS = 16_000 export function useStructuredAgentSessionOutbox(args: { @@ -36,7 +41,8 @@ export function useStructuredAgentSessionOutbox(args: { submissions: readonly AgentJournalSubmission[] }) { const { fence, sessionId, submissions, target } = args - const targetKey = target.kind === 'local' ? 'local' : `environment:${target.environmentId}` + // What resends, unblocks and drops a send in flight besides a Retry or a new send; see the hook. + const owner = useStructuredAgentSessionOutboxOwnerChange(target, fence) const [outbox, setOutbox] = useState(() => readMountedStructuredAgentSessionOutbox(sessionId, fence, readOutbox) ) @@ -68,19 +74,15 @@ export function useStructuredAgentSessionOutbox(args: { blockedIdRef.current = null retryWithFreshClientMessageIdRef.current = null probeAttemptsRef.current = { id: null, attempts: 0 } - }, [fence, sessionId, targetKey]) + }, [owner.ownerChange, owner.targetKey, sessionId]) useEffect(() => { const sessionChanged = outboxSessionRef.current !== sessionId outboxSessionRef.current = sessionId const current = sessionChanged - ? readMountedStructuredAgentSessionOutbox(sessionId, fence, readOutbox) + ? readMountedStructuredAgentSessionOutbox(sessionId, owner.fenceRef.current, readOutbox) : outboxRef.current - const next = current.map((entry) => - entry.state === 'dispatching' && !hasInFlightLaunchDispatch(entry, fence) - ? { ...entry, state: 'queued' as const } - : entry - ) + const next = requeueInterruptedStructuredAgentSessionDispatches(current, owner.fenceRef.current) if ( sessionChanged || next.some((entry, index) => entry !== current[index]) || @@ -90,7 +92,7 @@ export function useStructuredAgentSessionOutbox(args: { setOutbox(next) writeOutbox(sessionId, next) } - }, [fence, sessionId, target]) + }, [owner.fenceRef, owner.ownerChange, sessionId, target]) useEffect(() => { const current = outboxRef.current @@ -103,7 +105,7 @@ export function useStructuredAgentSessionOutbox(args: { .map((submission) => submission.clientMessageId) ) const next = reconcileStructuredAgentSessionOutbox(current, submissions) - const admittedInFlight = inFlightIdRef.current !== null && hostOwns.has(inFlightIdRef.current) + const admittedInFlight = journalAnswersInFlightSend(submissions, inFlightIdRef.current) if ( admittedInFlight || next.some((entry, index) => entry !== current[index]) || @@ -121,7 +123,10 @@ export function useStructuredAgentSessionOutbox(args: { dispatchGenerationRef.current += 1 inFlightIdRef.current = null } - if (blockedIdRef.current !== null && hostOwns.has(blockedIdRef.current)) { + const rejection = reconciledRejectionNotice(current, next, submissions) + if (rejection) { + setError(rejection) + } else if (blockedIdRef.current !== null && hostOwns.has(blockedIdRef.current)) { blockedIdRef.current = null setError(null) } else if ( @@ -240,7 +245,7 @@ export function useStructuredAgentSessionOutbox(args: { const probeSettled = probeId !== null && submissions.some((submission) => submission.clientMessageId === probeId) useEffect(() => { - if (probeId === null || probeSettled || fence === null) { + if (probeId === null || probeSettled || !owner.attached) { return } const attempts = probeAttemptsRef.current.id === probeId ? probeAttemptsRef.current.attempts : 0 @@ -257,7 +262,7 @@ export function useStructuredAgentSessionOutbox(args: { Math.min(UNCONFIRMED_PROBE_BASE_DELAY_MS * 2 ** attempts, UNCONFIRMED_PROBE_MAX_DELAY_MS) ) return () => clearTimeout(timer) - }, [fence, probeId, probeSettled, sessionId, targetKey]) + }, [owner.attached, owner.ownerChange, owner.targetKey, probeId, probeSettled, sessionId]) const send = useCallback( (text: string, attachments: readonly { path: string; previewUri: string }[] = []): boolean => { diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts index 7bf979f8ff0a..f0c3eaa59a83 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts @@ -20,10 +20,8 @@ export function useStructuredAgentSessionTransport(args: { enabled: providerVisible }) // A worktree switch hides the pane, but a message the user already sent is still owed a - // delivery. The read is what carries the journal rows that retire it, and its subscription - // takes a retaining hold — without which the host evicts the session once its idle window runs - // out and then refuses the send outright. Gated on `enabled`: a session not yet published has - // nothing to read. Attention is not the signal; owed work is. + // delivery, and the read is what carries the journal rows that retire it. Gated on `enabled`: + // a session not yet published has nothing to read. Attention is not the signal; owed work is. const hasUndelivered = useUndeliveredStructuredAgentSessionOutbox(sessionId) const read = useStructuredAgentSessionRead({ sessionId, diff --git a/src/renderer/src/i18n/en-runtime-required.json b/src/renderer/src/i18n/en-runtime-required.json index b5e8431c5738..650b98f7829a 100644 --- a/src/renderer/src/i18n/en-runtime-required.json +++ b/src/renderer/src/i18n/en-runtime-required.json @@ -2672,6 +2672,7 @@ "title": "Start a chat with {{value0}}" }, "error": { + "retryingSubtitle": "The transcript could not be read. Orca keeps trying to load it.", "subtitle": "The transcript could not be read. Toggle back to the terminal to keep working.", "title": "Could not load conversation" }, diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 2c9969e8b071..c761f300aa2f 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -17631,7 +17631,8 @@ }, "error": { "title": "Could not load conversation", - "subtitle": "The transcript could not be read. Toggle back to the terminal to keep working." + "subtitle": "The transcript could not be read. Toggle back to the terminal to keep working.", + "retryingSubtitle": "The transcript could not be read. Orca keeps trying to load it." }, "pairHost": "Pair a host to view agent chat history.", "notAgent": { diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 722d704aceee..77881925a522 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -14905,7 +14905,8 @@ }, "error": { "title": "No se pudo cargar la conversación", - "subtitle": "No se pudo leer la transcripción. Vuelva a la terminal para seguir trabajando." + "subtitle": "No se pudo leer la transcripción. Vuelva a la terminal para seguir trabajando.", + "retryingSubtitle": "No se pudo leer la transcripción. Orca sigue intentando cargarla." }, "pairHost": "Empareje un host para ver el historial de chat del agente.", "notAgent": { diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index add1dd437d3e..c0b0898d689d 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -17478,7 +17478,8 @@ }, "error": { "title": "Impossible de charger la conversation", - "subtitle": "La transcription n'a pas pu être lue. Rebasculez sur le terminal pour continuer à travailler." + "subtitle": "La transcription n'a pas pu être lue. Rebasculez sur le terminal pour continuer à travailler.", + "retryingSubtitle": "La transcription n'a pas pu être lue. Orca continue d'essayer de la charger." }, "pairHost": "Associez un hôte pour consulter l'historique de chat des agents.", "notAgent": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 7ffa3e0e2eb7..e89b94e9ee4b 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -17414,7 +17414,8 @@ }, "error": { "title": "会話を読み込めませんでした", - "subtitle": "トランスクリプトを読み取ることができませんでした。作業を続けるには、ターミナルに切り替えてください。" + "subtitle": "トランスクリプトを読み取ることができませんでした。作業を続けるには、ターミナルに切り替えてください。", + "retryingSubtitle": "トランスクリプトを読み取ることができませんでした。Orca は読み込みを再試行し続けます。" }, "pairHost": "ホストをペアリングして、Agent のチャット履歴を表示します。", "notAgent": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index b44e62130a5e..ab5d1d24d12e 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -17414,7 +17414,8 @@ }, "error": { "title": "대화를 로드할 수 없습니다.", - "subtitle": "스크립트를 읽을 수 없습니다. 계속 작업하려면 terminal로 다시 전환하세요." + "subtitle": "스크립트를 읽을 수 없습니다. 계속 작업하려면 terminal로 다시 전환하세요.", + "retryingSubtitle": "스크립트를 읽을 수 없습니다. Orca가 계속 다시 불러오기를 시도합니다." }, "pairHost": "호스트를 페어링하여 에이전트 채팅 기록을 확인하세요.", "notAgent": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 5e7a0c9ec0fd..1b2abbe0e58f 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -17379,7 +17379,8 @@ }, "error": { "title": "无法加载对话", - "subtitle": "无法读取记录。请切回终端继续工作。" + "subtitle": "无法读取记录。请切回终端继续工作。", + "retryingSubtitle": "无法读取记录。Orca 会继续尝试加载。" }, "pairHost": "配对主机以查看智能体聊天记录。", "notAgent": { diff --git a/src/renderer/src/runtime/local-structured-session-tabs-paint-window.test.ts b/src/renderer/src/runtime/local-structured-session-tabs-paint-window.test.ts new file mode 100644 index 000000000000..997e05b1ffcc --- /dev/null +++ b/src/renderer/src/runtime/local-structured-session-tabs-paint-window.test.ts @@ -0,0 +1,176 @@ +// @vitest-environment happy-dom + +// The window between first paint and the host's chat-tab inventory (F15a). The session writer may +// save in it, so the saved chat tabs must still be exactly where the user left them, and must stay +// there once the inventory lands. The fake host always answers `subscribeAll` at once with a +// snapshot that lacks both chats, as a host answering before its tab publication would. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import type { RuntimeMobileSessionTabsResult } from '../../../shared/runtime-types' +import type { Tab } from '../../../shared/tab-types' +import { buildPersistedUnifiedTabSessionData } from '../lib/workspace-session-unified-tabs' +import { useAppStore } from '../store' +import { + resetLocalStructuredSessionVersionForTests, + startLocalStructuredSessionTabsSync +} from './local-structured-session-tabs-sync' +import { resetWebSessionTabsSnapshotFreshnessForTests } from './web-session-tabs-sync' + +const WORKTREE_ID = 'repo-1::worktree-1' +const LEFT = 'left-group' +const RIGHT = 'right-group' +const EPOCH = 'structured:epoch-1' + +afterEach(() => { + resetLocalStructuredSessionVersionForTests() + resetWebSessionTabsSnapshotFreshnessForTests() +}) + +function tab(id: string, entityId: string, groupId: string, sortOrder: number): Tab { + const chat = id.startsWith('structured-agent-session-') + return { + id, + entityId, + groupId, + worktreeId: WORKTREE_ID, + contentType: chat ? 'agent-session' : 'terminal', + ...(chat ? { agentSessionAgent: 'codex' as const } : {}), + label: chat ? 'Codex Chat' : 'Terminal', + customLabel: null, + color: null, + sortOrder, + createdAt: sortOrder + 1 + } +} + +/** The saved session as hydrated: a terminal and one chat on the left, a chat on the right. */ +function hydrateSavedSession(): void { + const tabs = [ + tab('terminal-1', 'terminal-1', LEFT, 0), + tab('structured-agent-session-chat-left', 'chat-left', LEFT, 1), + tab('structured-agent-session-chat-right', 'chat-right', RIGHT, 2) + ] + useAppStore.setState({ + activeWorktreeId: WORKTREE_ID, + unifiedTabsByWorktree: { [WORKTREE_ID]: tabs }, + groupsByWorktree: { + [WORKTREE_ID]: [ + { + id: LEFT, + worktreeId: WORKTREE_ID, + activeTabId: 'terminal-1', + tabOrder: ['terminal-1', 'structured-agent-session-chat-left'] + }, + { + id: RIGHT, + worktreeId: WORKTREE_ID, + activeTabId: 'structured-agent-session-chat-right', + tabOrder: ['structured-agent-session-chat-right'] + } + ] + }, + layoutByWorktree: { + [WORKTREE_ID]: { + type: 'split', + direction: 'horizontal', + first: { type: 'leaf', groupId: LEFT }, + second: { type: 'leaf', groupId: RIGHT } + } + }, + activeGroupIdByWorktree: { [WORKTREE_ID]: RIGHT } + }) +} + +/** The tab layout the session writer would persist right now: which tabs, in which group and + * order, the split, and the active group. Field normalization an apply adds is not layout. */ +function persisted() { + const data = buildPersistedUnifiedTabSessionData(useAppStore.getState()) + return { + tabs: data.unifiedTabs?.[WORKTREE_ID]?.map(({ id, entityId, groupId, contentType }) => ({ + id, + entityId, + groupId, + contentType + })), + groups: data.tabGroups?.[WORKTREE_ID]?.map(({ id, activeTabId, tabOrder }) => ({ + id, + activeTabId, + tabOrder + })), + layout: data.tabGroupLayouts?.[WORKTREE_ID], + activeGroup: data.activeGroupIdByWorktree?.[WORKTREE_ID] + } +} + +function hostSnapshot(version: number, sessionIds: string[]): RuntimeMobileSessionTabsResult { + const ids = sessionIds.map((sessionId) => `agent-session:${sessionId}`) + return { + worktree: WORKTREE_ID, + publicationEpoch: EPOCH, + snapshotVersion: version, + activeGroupId: 'host-group', + activeTabId: ids[0] ?? null, + activeTabType: ids.length > 0 ? 'agent-session' : null, + tabGroups: [{ id: 'host-group', activeTabId: ids[0] ?? null, tabOrder: ids }], + tabs: sessionIds.map((sessionId, index) => ({ + type: 'agent-session' as const, + id: ids[index]!, + title: 'Codex Chat', + sessionId, + agent: 'codex' as const, + isActive: index === 0 + })) + } +} + +describe('chat tabs between first paint and the host inventory', () => { + it('saves the hydrated chat tabs unchanged before and after the inventory lands', async () => { + hydrateSavedSession() + const saved = persisted() + const inventory = Promise.withResolvers() + const priorApi = window.api + Object.defineProperty(window, 'api', { + configurable: true, + value: { + runtime: { + getStatus: vi.fn().mockResolvedValue({ + capabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + }), + call: vi.fn(() => inventory.promise), + subscribe: vi.fn(async (_args: unknown, callback: (response: unknown) => void) => { + callback({ + ok: true, + result: { type: 'snapshots', snapshots: [hostSnapshot(1, [])] } + }) + return { unsubscribe: vi.fn() } + }) + } + } + }) + let unsubscribe = (): void => {} + try { + const sync = startLocalStructuredSessionTabsSync({ + isDisposed: () => false, + setUnsubscribe: (next) => { + unsubscribe = next + } + }) + for (let turn = 0; turn < 20; turn += 1) { + await Promise.resolve() + } + expect(persisted()).toEqual(saved) + + inventory.resolve({ + ok: true, + result: { snapshots: [hostSnapshot(2, ['chat-left', 'chat-right'])] } + }) + await sync + + expect(persisted()).toEqual(saved) + } finally { + unsubscribe() + Object.defineProperty(window, 'api', { configurable: true, value: priorApi }) + } + }) +}) diff --git a/src/renderer/src/runtime/local-structured-session-tabs-sync-startup-restore.test.ts b/src/renderer/src/runtime/local-structured-session-tabs-sync-startup-restore.test.ts new file mode 100644 index 000000000000..e1c3ff805799 --- /dev/null +++ b/src/renderer/src/runtime/local-structured-session-tabs-sync-startup-restore.test.ts @@ -0,0 +1,50 @@ +// @vitest-environment happy-dom + +import { expect, it, vi } from 'vitest' +import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import { + restoreLocalStructuredSessionTabsOnce, + startLocalStructuredSessionTabsSync +} from './local-structured-session-tabs-sync' + +it('still subscribes when the startup restore it shares with hydration fails', async () => { + const priorApi = window.api + let answerFirstListing = (_response: unknown): void => undefined + const firstListing = new Promise((resolve) => { + answerFirstListing = resolve + }) + const call = vi + .fn() + .mockReturnValueOnce(firstListing) + .mockResolvedValue({ ok: true, result: { snapshots: [] } }) + const subscribe = vi.fn(async () => ({ unsubscribe: vi.fn() })) + Object.defineProperty(window, 'api', { + configurable: true, + value: { + runtime: { + getStatus: vi.fn().mockResolvedValue({ + capabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + }), + call, + subscribe + } + } + }) + try { + // Hydration fires the restore without waiting, then the sync joins that same attempt. + void restoreLocalStructuredSessionTabsOnce().catch(() => undefined) + const started = startLocalStructuredSessionTabsSync({ + isDisposed: () => false, + setUnsubscribe: () => undefined + }) + // The sync has joined the pending restore by now, so the failure reaches it too. + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(call).toHaveBeenCalledOnce() + answerFirstListing({ ok: false, error: { code: 'runtime_error', message: 'install failed' } }) + await started + + expect(subscribe).toHaveBeenCalledOnce() + } finally { + Object.defineProperty(window, 'api', { configurable: true, value: priorApi }) + } +}) diff --git a/src/renderer/src/runtime/local-structured-session-tabs-sync/subscription.ts b/src/renderer/src/runtime/local-structured-session-tabs-sync/subscription.ts index 29fc6230629b..0e773182edfd 100644 --- a/src/renderer/src/runtime/local-structured-session-tabs-sync/subscription.ts +++ b/src/renderer/src/runtime/local-structured-session-tabs-sync/subscription.ts @@ -41,7 +41,11 @@ export async function startLocalStructuredSessionTabsSync(args: { return } const supported = capabilities.includes(STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY) - await restoreLocalStructuredSessionTabsOnce(syncGeneration) + // Startup no longer waits on this restore, so its failure lands here: the subscription below + // re-asks the host, and its retry refreshes the inventory first. + await restoreLocalStructuredSessionTabsOnce(syncGeneration).catch((error: unknown) => { + console.warn('[structured-session-tabs] startup restore failed', error) + }) if (!isCurrent()) { return } diff --git a/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx new file mode 100644 index 000000000000..acdfdbb03233 --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx @@ -0,0 +1,51 @@ +// @vitest-environment happy-dom + +import { renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' + +const mocks = vi.hoisted(() => ({ supports: vi.fn() })) + +vi.mock('./runtime-rpc-client', () => ({ + runtimeEnvironmentSupportsCapability: mocks.supports +})) + +import { setLocalRuntimeCapabilitiesForTests } from './local-runtime-capabilities' +import { useStructuredAgentSessionHostAcceptsSend } from './structured-agent-session-accepted-send-capability' + +afterEach(() => { + setLocalRuntimeCapabilitiesForTests(null) + vi.clearAllMocks() +}) + +describe('whether a host accepts a send before any agent has it', () => { + it('reads the local host from its advertised capabilities', () => { + setLocalRuntimeCapabilitiesForTests([AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY]) + const { result } = renderHook(() => useStructuredAgentSessionHostAcceptsSend({ kind: 'local' })) + expect(result.current).toBe(true) + }) + + it('treats a local host that does not advertise it as an older one', () => { + setLocalRuntimeCapabilitiesForTests([]) + const { result } = renderHook(() => useStructuredAgentSessionHostAcceptsSend({ kind: 'local' })) + expect(result.current).toBe(false) + }) + + it('asks a remote host, and reads a failed probe as an older host', async () => { + mocks.supports.mockResolvedValueOnce(true).mockRejectedValueOnce(new Error('offline')) + const accepts = renderHook(() => + useStructuredAgentSessionHostAcceptsSend({ kind: 'environment', environmentId: 'env-1' }) + ) + await waitFor(() => expect(accepts.result.current).toBe(true)) + expect(mocks.supports).toHaveBeenCalledWith( + 'env-1', + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) + + const offline = renderHook(() => + useStructuredAgentSessionHostAcceptsSend({ kind: 'environment', environmentId: 'env-2' }) + ) + await waitFor(() => expect(mocks.supports).toHaveBeenCalledTimes(2)) + expect(offline.result.current).toBe(false) + }) +}) diff --git a/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts new file mode 100644 index 000000000000..c6e130b8bd5f --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts @@ -0,0 +1,87 @@ +import { useEffect, useLayoutEffect, useRef, useState, type RefObject } from 'react' +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import type { RuntimeClientTarget } from './runtime-client-target' +import { + ensureLocalRuntimeCapabilities, + readLocalRuntimeCapabilitiesOrUnknown +} from './local-runtime-capabilities' +import { runtimeEnvironmentSupportsCapability } from './runtime-rpc-client' + +function targetKey(target: RuntimeClientTarget): string { + return target.kind === 'local' ? 'local' : `environment:${target.environmentId}` +} + +/** + * Whether the host answers a send at acceptance and never refuses one because its agent could not + * start. Such a host records every send before it starts anything, so nothing about a moved fence + * calls for a resend. False until the host has said so: an older host is handled as it always was. + */ +export function useStructuredAgentSessionHostAcceptsSend(target: RuntimeClientTarget): boolean { + const key = targetKey(target) + const [answer, setAnswer] = useState<{ key: string; accepts: boolean }>(() => ({ + key, + accepts: + target.kind === 'local' && + (readLocalRuntimeCapabilitiesOrUnknown()?.includes( + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) ?? + false) + })) + const environmentId = target.kind === 'environment' ? target.environmentId : null + useEffect(() => { + let cancelled = false + const probe = + environmentId === null + ? ensureLocalRuntimeCapabilities().then( + (capabilities) => + capabilities?.includes(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) ?? false + ) + : runtimeEnvironmentSupportsCapability( + environmentId, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) + void probe + .catch(() => false) + .then((accepts) => { + if (!cancelled) { + setAnswer((current) => + current.key === key && current.accepts === accepts ? current : { key, accepts } + ) + } + }) + return () => { + cancelled = true + } + }, [environmentId, key]) + return answer.key === key && answer.accepts +} + +/** + * When an outbox treats its owner as changed: resending a send in flight under the same id, + * dropping that send's answer, and unblocking a refused head. An older host restarts the agent + * inside the send and refuses it, unrecorded, when that fails, so a new fence is its only word that + * another try may land. A host that accepts first records every send before it starts anything, + * so a moved fence means nothing there, and only a Retry or a new send goes out. + */ +export function useStructuredAgentSessionOutboxOwnerChange( + target: RuntimeClientTarget, + fence: number | null +): { + ownerChange: number | null + attached: boolean + fenceRef: RefObject + targetKey: string +} { + const acceptsSend = useStructuredAgentSessionHostAcceptsSend(target) + // Read by effects that run on an owner change, not on every fence move. + const fenceRef = useRef(fence) + useLayoutEffect(() => { + fenceRef.current = fence + }, [fence]) + return { + ownerChange: acceptsSend ? null : fence, + attached: fence !== null, + fenceRef, + targetKey: targetKey(target) + } +} diff --git a/src/shared/agent-session-error-text.test.ts b/src/shared/agent-session-error-text.test.ts new file mode 100644 index 000000000000..91fc1899d1d8 --- /dev/null +++ b/src/shared/agent-session-error-text.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { agentSessionErrorText } from './agent-session-error-text' + +describe('agentSessionErrorText', () => { + it('reads a stream failure payload by its message, not as [object Object]', () => { + expect(agentSessionErrorText({ code: 'runtime_error', message: 'journal unreadable' })).toBe( + 'journal unreadable' + ) + }) + + it('reads a thrown error by its message, without the class-name prefix', () => { + expect(agentSessionErrorText(new Error('journal read failed'))).toBe('journal read failed') + }) + + it('falls back to the code, then to the caller fallback', () => { + expect(agentSessionErrorText({ code: 'agent_session_ownership_unknown' })).toBe( + 'agent_session_ownership_unknown' + ) + expect(agentSessionErrorText({}, 'Request was not sent')).toBe('Request was not sent') + expect(agentSessionErrorText(undefined, 'Request was not sent')).toBe('Request was not sent') + expect(agentSessionErrorText('plain text')).toBe('plain text') + }) +}) diff --git a/src/shared/agent-session-error-text.ts b/src/shared/agent-session-error-text.ts new file mode 100644 index 000000000000..9709cd30a68f --- /dev/null +++ b/src/shared/agent-session-error-text.ts @@ -0,0 +1,22 @@ +// The one way a chat surface turns an agent-session failure into words. +// +// A unary RPC throws an `Error`, but a stream hands its failure over as the raw `{ code, message }` +// payload, and `String()` of that is `[object Object]` — which is what a failed chat used to show. + +export function agentSessionErrorText(error: unknown, fallback = 'Something went wrong.'): string { + if (error instanceof Error) { + return error.message || fallback + } + if (typeof error === 'string') { + return error || fallback + } + if (typeof error === 'object' && error !== null) { + if ('message' in error && typeof error.message === 'string' && error.message.length > 0) { + return error.message + } + if ('code' in error && typeof error.code === 'string' && error.code.length > 0) { + return error.code + } + } + return fallback +} diff --git a/src/shared/agent-session-journal-schemas.ts b/src/shared/agent-session-journal-schemas.ts index b7a80634bf84..04f7183eb3f6 100644 --- a/src/shared/agent-session-journal-schemas.ts +++ b/src/shared/agent-session-journal-schemas.ts @@ -304,7 +304,9 @@ export const AgentJournalSubmissionSchema = z.object({ reason: z.string().nullable(), submittedAt: z.number(), resolvedAt: z.number().nullable(), - recovered: z.literal(true).optional() + recovered: z.literal(true).optional(), + handoverRecorded: z.literal(true).optional(), + handedOverAt: z.number().optional() }) export function isAdmissibleAgentJournalItemBody(value: unknown): value is AgentJournalItemBody { diff --git a/src/shared/agent-session-journal-types.ts b/src/shared/agent-session-journal-types.ts index f5bd250c0c9c..0a6cb655cd80 100644 --- a/src/shared/agent-session-journal-types.ts +++ b/src/shared/agent-session-journal-types.ts @@ -364,6 +364,13 @@ export type AgentJournalSubmission = { /** Set when crash reconciliation resolved the dispatch, not the provider. A live * `unknown` is a send still outstanding; a recovered one outlived its writer. */ recovered?: true + /** The host accepted this send to hand over later; absent on sends dispatched as they were + * recorded (older hosts). With no `handedOverAt` yet, a pending one is still queued. */ + handoverRecorded?: true + /** When the host handed it to the provider (its `dispatch{pending}` row). */ + handedOverAt?: number + /** Host-only: the submission row's sequence, which tells which host process accepted it. */ + acceptedSequence?: number } /** Durable answer to "did my send land?", keyed by client message id. Only an diff --git a/src/shared/agent-session-mutation-envelope.ts b/src/shared/agent-session-mutation-envelope.ts index fe439c20b08e..f0cae00c1b34 100644 --- a/src/shared/agent-session-mutation-envelope.ts +++ b/src/shared/agent-session-mutation-envelope.ts @@ -91,6 +91,9 @@ export function admitAgentSessionMutation(input: { /** Decision from the durable ledger, evaluated under `hostFingerprint`. */ ledger: AgentSessionOperationDecision lease: AgentSessionLease + /** A write to the conversation, not to the provider child: a send is accepted and a Stop + * withdraws queued messages whoever owns the child, so the lease does not admit them. */ + conversationWrite?: true }): AgentSessionMutationAdmission { const { envelope, lease, ledger } = input const mismatch = agentSessionFingerprintConflict(envelope, input.hostFingerprint) @@ -109,6 +112,9 @@ export function admitAgentSessionMutation(input: { if (ledger.decision === 'replay') { return { decision: 'replay', row: ledger.row } } + if (input.conversationWrite) { + return { decision: 'admit', row: ledger.row } + } const leaseRefusal = refuseUnlessWriterAdmitted(lease) if (leaseRefusal) { return { decision: 'refused', refusal: leaseRefusal } diff --git a/src/shared/agent-session-queued-submission.ts b/src/shared/agent-session-queued-submission.ts new file mode 100644 index 000000000000..6295897fd54e --- /dev/null +++ b/src/shared/agent-session-queued-submission.ts @@ -0,0 +1,13 @@ +import type { AgentJournalSubmission } from './agent-session-journal-types' + +/** Accepted by the host and not yet handed to the provider: provably unwritten, so every way out + * of this state is delivery or a rejection, never doubt. */ +export function isQueuedAgentJournalSubmission( + submission: Pick +): boolean { + return ( + submission.handoverRecorded === true && + submission.dispatchState === 'pending' && + submission.handedOverAt === undefined + ) +} diff --git a/src/shared/agent-session-resume-marker.ts b/src/shared/agent-session-resume-marker.ts index 2c308ec998ec..3890bd8f81d3 100644 --- a/src/shared/agent-session-resume-marker.ts +++ b/src/shared/agent-session-resume-marker.ts @@ -6,8 +6,9 @@ // the stop, because that fact exists only in memory at that moment — the provider rewrites the // journal in its own words on reattach. // -// A marker has no expiry. It ends only by the user's own actions: a newer message in that chat, a -// successful resume, a dismissal, or closing the chat — each of which deletes it. +// A marker has no expiry. It ends when the chat's agent is started again other than by its own +// continuation, or by a successful resume, a dismissal, or closing the chat — each of which +// deletes it. import { z } from 'zod' import { AGENT_STATUS_STATES } from './agent-status-types' @@ -18,6 +19,7 @@ import { AGENT_SESSION_RESTART_ACTIVITY_MAX_TASKS } from './agent-session-restart-activity' import type { AgentSessionRestartActivity } from './agent-session-restart-activity' +import type { AgentJournalCursor } from './agent-session-journal-types' /** Why the app went away. Recorded because an update install is a restart the user did not choose, * and the surface that offers the resume says so. */ @@ -48,8 +50,9 @@ export type AgentSessionResumeMarker = { /** The work in flight when teardown observed it — a running turn, or a send that had not yet * become one. */ work: AgentSessionResumeWork - /** The user message observed at teardown; a newer one supersedes this offer before its turn opens. */ - latestUserItemId: string | null + /** The user message observed at teardown. Nothing reads it; still written for one release so the + * previous build, whose parser requires it, can read this marker after a downgrade. */ + latestUserItemId?: string | null /** Execution host's clock at teardown. */ recordedAt: number trigger: AgentSessionResumeTrigger @@ -65,6 +68,12 @@ export type AgentSessionResumeMarker = { providerHandleRoot: string /** Stable teardown identity for continuation deduplication, not launch ancestry. */ teardownId: string + /** + * Where the chat's journal stood when the offer was taken. A message accepted after it, or a + * journal on another epoch, means the chat moved on. Absent on markers from builds that did not + * record it. + */ + journalCursor?: AgentJournalCursor /** * What the session was doing, captured at the same stop-time snapshot that decided the offer. * The dialog row, the status bar and the wire candidate read ONLY this; nothing re-reads the @@ -114,11 +123,15 @@ const agentSessionRestartActivitySchema = z.object({ const agentSessionResumeMarkerSchema = z.object({ sessionId: markerField, work: agentSessionResumeWorkSchema, - latestUserItemId: markerField.nullable(), + latestUserItemId: markerField.nullable().optional(), recordedAt: z.number().int().nonnegative(), trigger: z.enum(AGENT_SESSION_RESUME_TRIGGERS), providerHandleRoot: markerField, teardownId: markerField, + journalCursor: z + .object({ epoch: markerField, sequence: z.number().int().nonnegative() }) + .optional() + .catch(undefined), activity: agentSessionRestartActivitySchema.optional().catch(undefined) }) @@ -129,6 +142,10 @@ export function parseAgentSessionResumeMarker(value: unknown): AgentSessionResum if (!parsed.success) { return null } - const { activity, ...marker } = parsed.data - return activity === undefined ? marker : { ...marker, activity } + const { activity, journalCursor, ...marker } = parsed.data + return { + ...marker, + ...(journalCursor === undefined ? {} : { journalCursor }), + ...(activity === undefined ? {} : { activity }) + } } diff --git a/src/shared/agent-session-wire.ts b/src/shared/agent-session-wire.ts index 7118a2237ebd..f5ceb625770d 100644 --- a/src/shared/agent-session-wire.ts +++ b/src/shared/agent-session-wire.ts @@ -295,7 +295,7 @@ export type AgentSessionAttachResult = { sessionId: string fence: number page: AgentSessionHistoryPage - /** Submissions the crash boundary settled as `unknown` while attaching. */ + /** Submissions a crash boundary left `unknown` that provider history could not decide. */ unconfirmedClientMessageIds: string[] /** The host-owned id of the tab showing this chat, when it has one. Absent from older hosts. */ tabId?: string diff --git a/src/shared/native-chat-empty-state.ts b/src/shared/native-chat-empty-state.ts index 2797648afd0f..3ef7b1a19025 100644 --- a/src/shared/native-chat-empty-state.ts +++ b/src/shared/native-chat-empty-state.ts @@ -19,6 +19,11 @@ export const NATIVE_CHAT_EMPTY_STATE_COPY = { title: 'Could not load conversation', subtitle: 'The transcript could not be read. Toggle back to the terminal to keep working.' }, + // The structured chat's read reconnects on its own; the terminal-backed read does not. + retryingError: { + title: 'Could not load conversation', + subtitle: 'The transcript could not be read. Orca keeps trying to load it.' + }, notAgent: { title: 'No conversation here', subtitle: 'This terminal is not running a recognized coding agent.' diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index db9d7f1b5443..dd992e32ea5b 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -165,6 +165,11 @@ export const STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY = 'agent-session.struct // clients skip the host's bounded best-effort settlement observation. export const AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY = 'agent-session.pending-send-result.v1' as const +// Why: a send is now answered once the host accepts it, before any agent has it. A client without +// this cannot show a message rejected after that answer, so the host holds its reply until the +// message is handed over or rejected. +export const AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY = + 'agent-session.accepted-send.v1' as const // Why: paired clients advertise Claude-structured support so the host can gate its agent-specific // journal and lifecycle surfaces independently from Codex support. export const CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY = @@ -302,6 +307,7 @@ export const NATIVE_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [ export const ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [ ...NATIVE_REMOTE_RUNTIME_CLIENT_CAPABILITIES, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, BROWSER_CLIENT_HOST_RUNTIME_CAPABILITY, BROWSER_CLIENT_PAGE_METADATA_RUNTIME_CAPABILITY, // Why: only the renderer runs the retirement-proof ledger; CLI and mobile must keep full lists. @@ -372,6 +378,9 @@ export const RUNTIME_CAPABILITIES = [ AGENT_SESSION_KEYBOARD_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + // The host side: it accepts a send before any agent has it, and a Stop with no writer before a + // turn starts, so a client may gate on either. + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RESUME_HISTORY_RUNTIME_CAPABILITY, diff --git a/src/shared/structured-agent-session-dispatch-rejection.ts b/src/shared/structured-agent-session-dispatch-rejection.ts index 14e197325d8f..a88dad5b16eb 100644 --- a/src/shared/structured-agent-session-dispatch-rejection.ts +++ b/src/shared/structured-agent-session-dispatch-rejection.ts @@ -29,6 +29,12 @@ export const DISPATCH_REJECTED_CODEX_QUEUE_FULL = 'codex structured dispatch que /** The provider confirmed a queued frame was withdrawn before execution. */ export const DISPATCH_REJECTED_CANCELLED = 'provider_cancelled_before_start' +/** Accepted by a host process that ended before handing it to any provider. */ +export const DISPATCH_REJECTED_HOST_RESTARTED = 'host_restarted_before_delivery' + +/** Accepted, then the provider was closed before the message was handed to it. */ +export const DISPATCH_REJECTED_PROVIDER_CLOSED = 'provider_closed_before_delivery' + export function dispatchWriteFailureReason(error: unknown): string { const detail = error instanceof Error ? error.message : String(error) return `${DISPATCH_REJECTED_WRITE_FAILED}: ${detail}` @@ -54,6 +60,8 @@ export function dispatchRejectionReasonIsInternal(reason: string | null | undefi dispatchRejectionWasTransportWriteFailure(reason) || reason === DISPATCH_REJECTED_QUEUE_FULL || reason === DISPATCH_REJECTED_CODEX_QUEUE_FULL || - reason === DISPATCH_REJECTED_CANCELLED + reason === DISPATCH_REJECTED_CANCELLED || + reason === DISPATCH_REJECTED_HOST_RESTARTED || + reason === DISPATCH_REJECTED_PROVIDER_CLOSED ) } diff --git a/src/shared/structured-agent-session-outbox.ts b/src/shared/structured-agent-session-outbox.ts index a336dea6fedd..1e6458a31b1b 100644 --- a/src/shared/structured-agent-session-outbox.ts +++ b/src/shared/structured-agent-session-outbox.ts @@ -7,7 +7,13 @@ import type { import { structuredAgentSessionPayloadFingerprint } from './structured-agent-session-mutation' import { DISPATCH_REJECTED_CANCELLED } from './structured-agent-session-dispatch-rejection' -export type StructuredAgentSessionOutboxState = 'queued' | 'dispatching' | 'unconfirmed' +/** `rejected`: the host settled the send as not delivered. The drain never sends it again on its + * own and nothing queues behind it; only the user's Retry does. */ +export type StructuredAgentSessionOutboxState = + | 'queued' + | 'dispatching' + | 'unconfirmed' + | 'rejected' export type StructuredAgentSessionOutboxEntry = { clientMessageId: string @@ -88,10 +94,12 @@ export function requeueStructuredAgentSessionSendRefusal( ) { return { ...entry, state: 'queued' } } + // Only here is the refusal proof the message never landed: an earlier attempt under this id, or + // one whose delivery was in doubt, may have, so those stay queued behind the block. return { ...entry, clientMessageId: createOperationId(), - state: 'queued', + state: 'rejected', lastAttemptAt: null, retryAfterUnknownSubmittedAt: null } @@ -116,6 +124,15 @@ export function reconcileStructuredAgentSessionOutbox( if (submission?.dispatchState === 'pending') { return entry.state === 'dispatching' ? [entry] : [{ ...entry, state: 'dispatching' as const }] } + // Accepted, then not delivered — the agent never started, or its start was refused. The text + // stays here for the user's Retry, and nothing queues behind it. `unconfirmed` is how a + // remount reads an entry it left dispatching; the journal has since answered it. + if ( + submission?.dispatchState === 'rejected' && + (entry.state === 'dispatching' || entry.state === 'unconfirmed') + ) { + return [{ ...entry, state: 'rejected' as const }] + } if ( submission?.dispatchState === 'unknown' && entry.retryAfterUnknownSubmittedAt !== -1 && @@ -147,6 +164,10 @@ export function admitStructuredAgentSessionOutboxEntry( blockedClientMessageId: string | null ): StructuredAgentSessionOutboxAdmission { for (const entry of entries) { + // It can no longer land, so nothing it could be reordered around; it waits for Retry. + if (entry.state === 'rejected') { + continue + } if (entry.state === 'unconfirmed' || entry.clientMessageId === blockedClientMessageId) { return { state: 'blocked', entry } } @@ -176,7 +197,7 @@ export function parseStructuredAgentSessionOutboxEntry( !Array.isArray(body.blocks) || !Array.isArray(entry.previewUris) || !entry.previewUris.every((uri) => typeof uri === 'string') || - !['queued', 'dispatching', 'unconfirmed'].includes(entry.state ?? '') + !['queued', 'dispatching', 'unconfirmed', 'rejected'].includes(entry.state ?? '') ) { return null } diff --git a/src/shared/structured-agent-session-read-refusal.ts b/src/shared/structured-agent-session-read-refusal.ts index 18487f12be38..b604c98113a0 100644 --- a/src/shared/structured-agent-session-read-refusal.ts +++ b/src/shared/structured-agent-session-read-refusal.ts @@ -24,6 +24,19 @@ export const AGENT_SESSION_UNATTACHED_REFUSAL_CODE = 'agent_session_ownership_un */ export const AGENT_SESSION_UNATTACHED_READ_GRACE_MS = 5_000 +/** Raised for a chat whose journal file is not a usable database. */ +export const AGENT_SESSION_JOURNAL_UNREADABLE_REFUSAL_CODE = 'agent_session_journal_unreadable' + +function isReadRefusal(error: unknown, code: string): boolean { + if (typeof error === 'string') { + return error === code + } + if (typeof error !== 'object' || error === null) { + return false + } + return ('code' in error && error.code === code) || ('message' in error && error.message === code) +} + /** * Whether a read failure is that refusal. * @@ -31,15 +44,22 @@ export const AGENT_SESSION_UNATTACHED_READ_GRACE_MS = 5_000 * bare refusal code, and the raw failure payload a stream delivers to its error callback. */ export function isUnattachedAgentSessionReadRefusal(error: unknown): boolean { + return isReadRefusal(error, AGENT_SESSION_UNATTACHED_REFUSAL_CODE) +} + +/** + * A read refusal no retry can change: re-asking reads the same unusable file. Matched on the + * message, which a read raises as the bare code: a send's refusal under the same code carries its + * cause as the message instead, and that cause can clear. + */ +export function isFinalAgentSessionReadRefusal(error: unknown): boolean { if (typeof error === 'string') { - return error === AGENT_SESSION_UNATTACHED_REFUSAL_CODE - } - if (typeof error !== 'object' || error === null) { - return false + return error === AGENT_SESSION_JOURNAL_UNREADABLE_REFUSAL_CODE } - const { code, message } = error as { code?: unknown; message?: unknown } return ( - code === AGENT_SESSION_UNATTACHED_REFUSAL_CODE || - message === AGENT_SESSION_UNATTACHED_REFUSAL_CODE + typeof error === 'object' && + error !== null && + 'message' in error && + error.message === AGENT_SESSION_JOURNAL_UNREADABLE_REFUSAL_CODE ) } diff --git a/src/shared/structured-agent-session-saved-status.test.ts b/src/shared/structured-agent-session-saved-status.test.ts new file mode 100644 index 000000000000..5601ad189fe8 --- /dev/null +++ b/src/shared/structured-agent-session-saved-status.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest' +import type { + AgentJournalDispatchState, + AgentJournalRenderItem, + AgentJournalSubmission +} from './agent-session-journal-types' +import { projectStructuredAgentSessionStatusSummary } from './structured-agent-session-projection' +import { + isSavableStructuredAgentSessionProjection, + parseStructuredAgentSessionSavedProjection, + STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION +} from './structured-agent-session-saved-status' + +function item( + itemId: string, + sequence: number, + body: AgentJournalRenderItem['body'] +): AgentJournalRenderItem { + return { itemId, sequence, revision: 1, observedAt: sequence * 1_000, body } +} + +function submission( + clientMessageId: string, + fence: number, + dispatchState: AgentJournalDispatchState +): AgentJournalSubmission { + return { + clientMessageId, + fence, + payloadFingerprint: clientMessageId, + dispatchState, + providerItemId: null, + reason: null, + submittedAt: fence * 1_000, + resolvedAt: dispatchState === 'pending' ? null : fence * 1_000 + 1 + } +} + +const settledTurn: AgentJournalRenderItem[] = [ + item('user-1', 1, { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Fix it' }] }), + item('turn-1', 2, { + kind: 'turn', + turnId: 'turn-1', + state: 'completed', + startedAt: 2_000, + completedAt: 5_000 + }), + item('reply-1', 3, { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: 'Fixed.' }] + }) +] + +describe('saved chat status', () => { + // A saved copy is trusted across builds only while this output is unchanged. If this fails, + // bump STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, then update the snapshot (C14). + it('pins what a settled chat projects under the current projection version', () => { + expect({ + version: STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION, + projection: projectStructuredAgentSessionStatusSummary(settledTurn) + }).toMatchInlineSnapshot(` + { + "projection": { + "lastAssistantMessage": "Fixed.", + "latestPrompt": "Fix it", + "status": "idle", + "statusStartedAt": 5000, + }, + "version": 1, + } + `) + }) + + // What makes a saved copy fence-free: a settled projection can only stay settled as the fence + // rises, because a higher fence only stops older sends from counting (C9). + it('keeps every idle or no-turn projection unchanged as the fence rises', () => { + const states: AgentJournalDispatchState[] = ['pending', 'accepted', 'rejected', 'unknown'] + const histories: AgentJournalRenderItem[][] = [[], settledTurn] + let checked = 0 + for (const items of histories) { + for (const first of states) { + for (const second of states) { + for (const firstFence of [1, 2, 3]) { + const submissions = [ + submission('send-1', firstFence, first), + submission('send-2', 1, second) + ] + for (const fence of [1, 2, 3]) { + const settled = projectStructuredAgentSessionStatusSummary(items, submissions, fence) + if (!isSavableStructuredAgentSessionProjection(settled)) { + continue + } + checked += 1 + for (const later of [fence + 1, fence + 5]) { + expect( + projectStructuredAgentSessionStatusSummary(items, submissions, later) + ).toEqual(settled) + } + } + } + } + } + } + expect(checked).toBeGreaterThan(0) + }) + + it('round-trips a settled projection and refuses anything a saved copy never holds', () => { + const settled = projectStructuredAgentSessionStatusSummary(settledTurn) + expect(parseStructuredAgentSessionSavedProjection(JSON.stringify(settled))).toEqual(settled) + expect( + parseStructuredAgentSessionSavedProjection(JSON.stringify({ status: null, latestPrompt: '' })) + ).toEqual({ status: null, latestPrompt: '' }) + for (const unsaved of [ + { ...settled, status: 'working' }, + { ...settled, toolName: 'Read' }, + { ...settled, turnOutcome: 'not-an-outcome' }, + { status: 'idle' } + ]) { + expect(parseStructuredAgentSessionSavedProjection(JSON.stringify(unsaved))).toBeNull() + } + expect(parseStructuredAgentSessionSavedProjection('{not json')).toBeNull() + }) +}) diff --git a/src/shared/structured-agent-session-saved-status.ts b/src/shared/structured-agent-session-saved-status.ts new file mode 100644 index 000000000000..998ef1483feb --- /dev/null +++ b/src/shared/structured-agent-session-saved-status.ts @@ -0,0 +1,71 @@ +// A chat's last settled status, saved so a restarted host can list it without replaying its journal. +// +// Only an idle or no-turn projection is saved, keyed by the journal position it was computed at. +// A restart trusts it only when the journal still stands exactly there; anything else is a miss and +// the journal is opened, so a saved copy can go stale but can never be shown stale. + +import type { AgentJournalCursor } from './agent-session-journal-types' +import { isAgentJournalTurnOutcome } from './agent-turn-outcome' +import type { StructuredAgentSessionStatusProjection } from './structured-agent-session-projection' + +/** Bump whenever the projection or the reducer changes what an idle or no-turn chat projects. */ +export const STRUCTURED_AGENT_SESSION_STATUS_PROJECTION_VERSION = 1 + +export type StructuredAgentSessionSavedStatus = { + v: number + /** Where the journal stood when `projection` was computed. */ + cursor: AgentJournalCursor + projection: StructuredAgentSessionStatusProjection + /** The journal's `lastActivityAt()` at that position, which dates the row. */ + lastActivityAt: number +} + +/** Whether a projection is one a saved copy may hold: settled, so no fence can change it. */ +export function isSavableStructuredAgentSessionProjection( + projection: StructuredAgentSessionStatusProjection +): boolean { + return projection.status === null || projection.status === 'idle' +} + +function optionalString(value: unknown): boolean { + return value === undefined || typeof value === 'string' +} + +/** The saved projection, or null for anything this build did not write: a miss, never an error. */ +export function parseStructuredAgentSessionSavedProjection( + json: string +): StructuredAgentSessionStatusProjection | null { + let raw: unknown + try { + raw = JSON.parse(json) + } catch { + return null + } + if (typeof raw !== 'object' || raw === null) { + return null + } + const value: Record = { ...raw } + const { status, latestPrompt, lastAssistantMessage, turnOutcome, statusStartedAt } = value + if (status !== null && status !== 'idle') { + return null + } + if (typeof latestPrompt !== 'string' || !optionalString(lastAssistantMessage)) { + return null + } + if (value.toolName !== undefined || value.toolInput !== undefined) { + return null + } + if (turnOutcome !== undefined && !isAgentJournalTurnOutcome(turnOutcome)) { + return null + } + if (statusStartedAt !== undefined && typeof statusStartedAt !== 'number') { + return null + } + return { + status, + latestPrompt, + ...(typeof lastAssistantMessage === 'string' ? { lastAssistantMessage } : {}), + ...(isAgentJournalTurnOutcome(turnOutcome) ? { turnOutcome } : {}), + ...(typeof statusStartedAt === 'number' ? { statusStartedAt } : {}) + } +} diff --git a/src/shared/structured-agent-session-send-disposition.ts b/src/shared/structured-agent-session-send-disposition.ts index 16765a024714..72368dae921f 100644 --- a/src/shared/structured-agent-session-send-disposition.ts +++ b/src/shared/structured-agent-session-send-disposition.ts @@ -7,6 +7,7 @@ // the refs, the React state and the storage write, and nothing else decides an // entry's state. +import type { AgentJournalSubmission } from './agent-session-journal-types' import type { AgentSessionMutationResult, AgentSessionSendResult } from './agent-session-wire' import { dispatchRejectionReasonIsInternal, @@ -72,6 +73,40 @@ function refusedRedelivery( ) } +/** Whether the journal already answers a send still in flight, so its own reply adds nothing: the + * host holds the message, or rejected it — a later `pending` reply must not undo that. */ +export function journalAnswersInFlightSend( + submissions: readonly AgentJournalSubmission[], + clientMessageId: string | null +): boolean { + return submissions.some( + (submission) => + submission.clientMessageId === clientMessageId && submission.dispatchState !== 'unknown' + ) +} + +/** What to say for a message the reconcile just settled as accepted and then not delivered, which + * keeps its Retry; null when there is none. A Stop's withdrawal is dropped there, so says nothing. */ +export function reconciledRejectionNotice( + previous: readonly StructuredAgentSessionOutboxEntry[], + reconciled: readonly StructuredAgentSessionOutboxEntry[], + submissions: readonly AgentJournalSubmission[] +): string | null { + const was = ( + entries: readonly StructuredAgentSessionOutboxEntry[], + clientMessageId: string, + state: StructuredAgentSessionOutboxEntry['state'] + ): boolean => + entries.some((entry) => entry.clientMessageId === clientMessageId && entry.state === state) + const rejected = submissions.find( + (submission) => + submission.dispatchState === 'rejected' && + was(previous, submission.clientMessageId, 'dispatching') && + was(reconciled, submission.clientMessageId, 'rejected') + ) + return rejected ? structuredAgentSessionRejectionNotice(rejected.reason) : null +} + /** * What to put on screen for a rejection. * @@ -124,12 +159,17 @@ export function disposeStructuredAgentSessionSendResult( ) : candidate ) + const refused = entries[refusedIndex] return { entries, error: result.refusal.message, // Read back by index rather than from the input: a refusal can rotate the id, and the // refused entry is not always the head now that an admitted one no longer holds the queue. - blockedClientMessageId: entries[refusedIndex]?.clientMessageId ?? null, + // A rejected one holds nothing: it can no longer land, and it keeps its own Retry. + blockedClientMessageId: + !refused || refused.state === 'rejected' + ? input.blockedClientMessageId + : refused.clientMessageId, retryWithFreshClientMessageId: null } } @@ -152,9 +192,9 @@ export function disposeStructuredAgentSessionSendResult( } if (submission.dispatchState === 'rejected') { return { - entries: replaceEntryState(input, 'queued'), + entries: replaceEntryState(input, 'rejected'), error: structuredAgentSessionRejectionNotice(submission.reason), - blockedClientMessageId: input.entry.clientMessageId, + blockedClientMessageId: input.blockedClientMessageId, retryWithFreshClientMessageId: input.entry.clientMessageId } } diff --git a/src/shared/structured-agent-session-unanswered-dispatch.ts b/src/shared/structured-agent-session-unanswered-dispatch.ts index 36c70a357cb7..002db36cb57d 100644 --- a/src/shared/structured-agent-session-unanswered-dispatch.ts +++ b/src/shared/structured-agent-session-unanswered-dispatch.ts @@ -1,4 +1,5 @@ import type { AgentJournalSubmission } from './agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from './agent-session-queued-submission' /** One send the provider has neither opened a turn for nor refused; the rule is explained on * `hasUnansweredStructuredAgentSessionDispatch`, which asks it of every send. */ @@ -6,6 +7,10 @@ export function isUnansweredStructuredAgentSessionDispatch( submission: AgentJournalSubmission, currentFence?: number | null ): boolean { + if (isQueuedAgentJournalSubmission(submission)) { + // Accepted and still owed to whichever child the host starts next, whatever the fence. + return true + } return ( (currentFence == null || submission.fence >= currentFence) && (submission.dispatchState === 'pending' || diff --git a/tests/e2e/codex-child-approval-activity-row.unit.test.ts b/tests/e2e/codex-child-approval-activity-row.unit.test.ts index b9210bf79e87..71bb0fddce18 100644 --- a/tests/e2e/codex-child-approval-activity-row.unit.test.ts +++ b/tests/e2e/codex-child-approval-activity-row.unit.test.ts @@ -143,7 +143,9 @@ async function openHost() { journalDir: join(root, SESSION) }) const feed = new StructuredAgentSessionStatusFeed({ - sessions: new Map([[SESSION, indexedStatusFeedSession({ journal, hasProviderChild: true })]]), + sessions: new Map([ + [SESSION, indexedStatusFeedSession({ journal, child: { phase: 'ready' } })] + ]), getRecord: () => null, now: () => 1, readBackgroundTasks: () => ({ state: 'monitoring', tasks: [] }) diff --git a/tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts new file mode 100644 index 000000000000..f7f23d528aaf --- /dev/null +++ b/tests/e2e/cross-version-wire/agent-session-resume-marker-downgrade.unit.test.ts @@ -0,0 +1,52 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, test } from 'vitest' +import { structuredAgentSessionWorkingAtStop } from '../../../src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown' +import { + journal, + NOW, + record, + SESSION, + TEARDOWN_CURRENT, + turnItem +} from '../../../src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness' +import { AgentSessionRecoveryCapsule } from '../../../src/main/runtime/agent-session-recovery-capsule' +import { importReleaseCheckoutModule, materializeReleaseCheckout } from './release-checkout' + +// A release whose marker parser still requires the chat's newest message id. +const BASELINE_REF = 'v1.4.211' + +test('an older build reads the restart offer this build records at quit', async () => { + const directory = mkdtempSync(join(tmpdir(), 'orca-resume-marker-downgrade-')) + try { + // This build: the offer teardown takes for a chat whose turn was running. + const marker = structuredAgentSessionWorkingAtStop({ + sessionId: SESSION, + session: { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }, + getRecord: () => record(), + backgroundTasks: () => undefined, + trigger: 'quit', + teardownId: TEARDOWN_CURRENT, + now: NOW + }) + if (!marker) { + throw new Error('this build recorded no offer for a running turn') + } + await new AgentSessionRecoveryCapsule(directory).record([marker], NOW) + + // The older build, after a downgrade, still lists it. + const checkout = await materializeReleaseCheckout(BASELINE_REF) + const baseline = await importReleaseCheckoutModule( + checkout, + 'src/main/runtime/agent-session-recovery-capsule.ts' + ) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pinned release exports this class with the constructor and `list` called below; a missing one fails the test. + const OldCapsule = baseline.AgentSessionRecoveryCapsule as new (directory: string) => { + list: (now: number) => Promise<{ sessionId: string }[]> + } + expect(await new OldCapsule(directory).list(NOW)).toMatchObject([{ sessionId: SESSION }]) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts index b038a558517f..2c313128d69b 100644 --- a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts +++ b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts @@ -22,6 +22,7 @@ import { AgentSessionRecordStore } from '../../../src/main/runtime/agent-session import { RuntimeSubscriptionRegistry } from '../../../src/main/runtime/runtime-subscription-registry' import type { AgentSessionSubscribeEvent } from '../../../src/shared/agent-session-wire' import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, AGENT_SESSION_QUESTION_ANSWERS_RUNTIME_CAPABILITY, AGENT_SESSION_REWIND_RUNTIME_CAPABILITY, @@ -619,6 +620,10 @@ describe('cross-version structured agent sessions', () => { let store: AgentSessionRecordStore let runtime: unknown + /** Holds a provider start open, so a reply's timing can be read against it. */ + let startGate: Promise = Promise.resolve() + let starts = 0 + /** Phase 2 owns provider processes; the adapter is the only stub here. */ function adapter(): StructuredAgentSessionAdapter { return { @@ -626,23 +631,27 @@ describe('cross-version structured agent sessions', () => { // `supportsLocation`, which this fake also lacks, so the client-supplied-location gate // refused for the fake's silence rather than for the location. supportsCreate: () => true, - acquire: async ({ fence }) => ({ - process: { - hostId: 'local', - pid: 4242, - processStartTimeMs: 1_700_000_000_000, - spawnToken: store.getRecord(SESSION)?.lease.reservedSpawnToken ?? 'spawn-a' - }, - link: { - linkId: `link-${fence}`, - handle: { provider: 'codex', threadId: THREAD }, - // A restarted host re-proves the thread it inherited; only the first - // owner of a session may claim to have created it. - origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', - mintedAtFence: fence, - observedAt: NOW + acquire: async ({ fence }) => { + starts += 1 + await startGate + return { + process: { + hostId: 'local', + pid: 4242, + processStartTimeMs: 1_700_000_000_000, + spawnToken: store.getRecord(SESSION)?.lease.reservedSpawnToken ?? 'spawn-a' + }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + // A restarted host re-proves the thread it inherited; only the first + // owner of a session may claim to have created it. + origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', + mintedAtFence: fence, + observedAt: NOW + } } - }), + }, dispatch: async () => ({ state: 'accepted', providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 1 } @@ -697,14 +706,18 @@ describe('cross-version structured agent sessions', () => { return reattached } - async function call(method: string, params: unknown): Promise { + async function call( + method: string, + params: unknown, + clientCapabilities: readonly string[] = [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + ): Promise { return callBuild( current, method, params, { clientKind: 'runtime', - clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY], + clientCapabilities, clientId: 'paired-device-1', connectionId: 'connection-1' }, @@ -723,6 +736,7 @@ describe('cross-version structured agent sessions', () => { beforeEach(async () => { resetOperationIds() + startGate = Promise.resolve() root = await mkdtemp(join(tmpdir(), 'orca-cross-version-agent-session-')) runtime = runtimeStub() await bootHost('a') @@ -741,9 +755,12 @@ describe('cross-version structured agent sessions', () => { const held = first.cursor const restarted = await bootHost('b') - await restarted.restoreReadableSessions() - // Restart restores the session for READING. The chat the client still has open takes its - // hold, and that is what gives the session a provider child again. + // What every boot runs before its startup pass; it opens no journal. The hold and the + // reads open it on first use, and the hold gives the session a child again. + await restarted.reconcileRestartLeases() + // The first read after the restart opens the journal on its own. + const reread = await answer('agentSession.history', { sessionId: SESSION, direction: 'tail' }) + expect(JSON.stringify(reread)).toContain('before restart') await answer('agentSession.hold', { sessionId: SESSION, holderId: 'surface-1' }) const resumedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 expect(resumedFence).toBeGreaterThan(created.fence) @@ -776,5 +793,58 @@ describe('cross-version structured agent sessions', () => { fence: reattached.fence }) }) + + // A released client answers a send's `pending` as delivered-or-refused; it has no way to show + // a rejection that arrives after it. So it is answered once the message is handed over, while + // a client that advertises accepted sends is answered at acceptance, start or no start (W9). + it('holds the send reply of a released client until the handover, and answers a current one at once', async () => { + const released = [ + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, + AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY + ] + expect(baseline.capabilities).not.toContain(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) + const created = await answer('agentSession.create', createIntentParams()) + await bootHost('b') + let open = (): void => undefined + startGate = new Promise((resolve) => (open = resolve)) + + let answered = false + const releasedReply = call( + 'agentSession.send', + sendParams('released', created.fence), + released + ).finally(() => (answered = true)) + // The start that delivers it is under way, and the reply still waits for it. + const before = starts + await vi.waitFor(() => expect(starts).toBeGreaterThan(before)) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(answered).toBe(false) + open() + const [reply] = await releasedReply + // Handed over, whatever the provider has said since. + expect(reply).toMatchObject({ + ok: true, + result: { value: { submission: { handedOverAt: expect.any(Number) } } } + }) + + const restarted = await bootHost('c') + startGate = new Promise((resolve) => (open = resolve)) + const currentReply = await call('agentSession.send', sendParams('current', created.fence), [ + ...released, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ]) + expect(currentReply[0]).toMatchObject({ + ok: true, + result: { value: { submission: { dispatchState: 'pending', handoverRecorded: true } } } + }) + open() + await vi.waitFor(async () => + expect( + (await restarted.journalSnapshot(SESSION)).submissions.every( + (row) => row.dispatchState !== 'pending' || row.handedOverAt + ) + ).toBe(true) + ) + }) }) }) diff --git a/tests/e2e/cross-version-wire/structured-agent-session-surface-manifest.ts b/tests/e2e/cross-version-wire/structured-agent-session-surface-manifest.ts index d8c8b80b3293..f827d3cda5f7 100644 --- a/tests/e2e/cross-version-wire/structured-agent-session-surface-manifest.ts +++ b/tests/e2e/cross-version-wire/structured-agent-session-surface-manifest.ts @@ -115,7 +115,8 @@ export const STRUCTURED_CALLS: { hostMethod: 'revealSession', result: { ok: true, sessionId: SESSION, workspaceId: WORKSPACE, agent: 'codex', readable: true } }, - { method: 'agentSession.hold', hostMethod: 'hold', result: { held: true } }, + // A no-op on a host that starts an agent only for work; it still builds the host. + { method: 'agentSession.hold', hostMethod: null, result: { held: true } }, // The restart-resume surface. Bare additions, not capability-negotiated: an RPC method's // absence is explicit (`method_not_found`), which the old-dispatcher case below asserts, so a // newer client learns it during negotiation instead of by being met with silence. @@ -129,17 +130,14 @@ export const STRUCTURED_CALLS: { hostMethod: 'restartResumableDismiss', result: { dismissed: 0 } }, - { - method: 'agentSession.restartResume', - hostMethod: 'restartResumeAll', - result: { results: [] } - }, + // Reattaching alone is nothing now, so this answers that nothing was resumed. + { method: 'agentSession.restartResume', hostMethod: null, result: { results: [] } }, { method: 'agentSession.restartContinue', hostMethod: 'restartContinueAll', result: { resumed: [], continued: [] } }, - { method: 'agentSession.release', hostMethod: 'release', result: { released: true } }, + { method: 'agentSession.release', hostMethod: null, result: { released: true } }, { method: 'agentSession.history', hostMethod: 'history', diff --git a/tests/e2e/structured-chat-owner-status-activation.unit.test.ts b/tests/e2e/structured-chat-owner-status-activation.unit.test.ts index f2c7cbd39d45..1d541819e4b3 100644 --- a/tests/e2e/structured-chat-owner-status-activation.unit.test.ts +++ b/tests/e2e/structured-chat-owner-status-activation.unit.test.ts @@ -136,12 +136,13 @@ describe('a chat at rest keeps its worktree activatable', () => { expect(await activate()).toBe('structured') }) - it('after an app restart restored it for reading', async () => { + it('after an app restart settled it without keeping it open', async () => { + await host.setSessionTabVisibility(SESSION, true) await host.flushAllStreamedEvents() store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) openHost() - await host.restoreReadableSessions() - expect(host.hasSession(SESSION)).toBe(true) + await host.restoreStartupSessions() + expect(host.hasSession(SESSION)).toBe(false) expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', ownerProcess: null