From ea3ee4fcc2cad69761d4853a3bdb1e4be5aea7df Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:12:58 -0700 Subject: [PATCH 01/24] fix(native-chat): a turn a proven crash cut short reads interrupted, ending when it was last seen working --- ...gent-session-dead-generation-settlement.ts | 15 +++-- ...d-agent-session-stale-turn-verdict.test.ts | 40 ++++++++--- ...ctured-agent-session-stale-turn-verdict.ts | 39 +++++++++-- ...t-session-wedged-profile-migration.test.ts | 67 +++++++++++++++++-- 4 files changed, 134 insertions(+), 27 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 05f7b4a7fe84..6a7a2725ea53 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -206,9 +206,9 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { /** * Settles whatever a generation with no child in this process left running: found when a new child * is acquired, or when a chat is reopened for reading. Derived from the journal and the lease's - * death evidence each time, so nothing is owed in between. Only an observed exit earns an end time - * and the exit copy. Must run before a new child's buffered events land, or a live turn would be - * judged. + * death evidence each time, so nothing is owed in between. Proven death ends the turn interrupted, + * and only a watched exit carries its reason into the copy. Must run before a new child's buffered + * events land, or a live turn would be judged. */ export async function settleStaleStructuredAgentSessionState(input: { journal: AgentSessionJournal @@ -219,7 +219,7 @@ export async function settleStaleStructuredAgentSessionState(input: { }): Promise { const { journal } = input const items = journal.snapshot().items - const verdict = turnVerdictFromDeathEvidence(input.deathEvidence) + const verdict = turnVerdictFromDeathEvidence(input.deathEvidence, items) const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` const mutations: JournalLifecycleMutationInput[] = [] @@ -237,7 +237,12 @@ export async function settleStaleStructuredAgentSessionState(input: { identity: { provider: 'orca', clientMessageId: settlementId }, body: { kind: 'status', - text: boundJournalStatusText(unexpectedProviderExitOutcome(input.deathEvidence?.detail)) + // Only a watched exit carries the provider's own reason; a probe's detail is Orca's. + text: boundJournalStatusText( + unexpectedProviderExitOutcome( + input.deathEvidence?.kind === 'exit-observed' ? input.deathEvidence.detail : undefined + ) + ) } }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 4ab7b3ad010c..86ba7b5dab47 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -75,17 +75,39 @@ function promptItem(state: 'pending' | 'resolved', sequence: number): AgentJourn } describe('turn verdict from death evidence', () => { - it('earns an end time only from an observed exit', () => { + // A running turn first seen at 100, a tool call at 300, and a row crash recovery wrote at 450. + const items: AgentJournalRenderItem[] = [ + lifecycleItem('turn-2', 'running', 100), + { ...lifecycleItem('turn-tool', 'running', 300) }, + { ...lifecycleItem('turn-recovered', 'completed', 450), recovered: true, recoveredAt: 450 } + ] + + it('ends a watched exit at the exit', () => { expect( - turnVerdictFromDeathEvidence({ kind: 'exit-observed', detail: 'exit', observedAt: 500 }) + turnVerdictFromDeathEvidence( + { kind: 'exit-observed', detail: 'exit', observedAt: 500 }, + items + ) ).toEqual({ state: 'interrupted', completedAt: 500 }) - expect( - turnVerdictFromDeathEvidence({ kind: 'pid-absent', detail: 'gone', observedAt: 500 }) - ).toEqual({ state: 'unverifiable' }) - expect( - turnVerdictFromDeathEvidence({ kind: 'identity-mismatch', detail: 'pid', observedAt: 500 }) - ).toEqual({ state: 'unverifiable' }) - expect(turnVerdictFromDeathEvidence(null)).toEqual({ state: 'unverifiable' }) + }) + + it.each(['pid-absent', 'identity-mismatch'] as const)( + 'ends a %s proof at the last thing the journal saw live, not at the probe', + (kind) => { + // Probed at 9000, long after the crash: the downtime is never counted as work. + expect( + turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, items) + ).toEqual({ state: 'interrupted', completedAt: 300 }) + // Nothing live before the probe: the probe is the only bound there is. + expect(turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 50 }, items)).toEqual( + { state: 'interrupted', completedAt: 50 } + ) + } + ) + + it('leaves a release nothing proved unverifiable', () => { + expect(turnVerdictFromDeathEvidence(null, items)).toEqual({ state: 'unverifiable' }) + expect(turnVerdictFromDeathEvidence(undefined, items)).toEqual({ state: 'unverifiable' }) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index b6e9b9bde5ea..30707cef6152 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -1,8 +1,9 @@ // What the host may durably say about a turn whose provider child is gone. // -// `interrupted` requires the host to have seen the child exit; that receipt is the only end time it -// is allowed to record. Everything weaker — a pid probe, an identity mismatch, a journal found -// running with no observed exit on the record — is `unverifiable` and carries no end at all. +// `interrupted` requires proof the child is gone, which is exactly when a lease carries death +// evidence: a watched exit, or a local probe that found the recorded pid gone or reused. A release +// nothing proved — lost contact, an unverifiable identity, a stop that outlived the ladder — carries +// none, and its turn is `unverifiable` with no end at all. import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { @@ -25,11 +26,35 @@ export const UNVERIFIABLE_TURN_VERDICT: StructuredAgentSessionTurnVerdict = { } export function turnVerdictFromDeathEvidence( - evidence: AgentSessionDeathEvidence | null | undefined + evidence: AgentSessionDeathEvidence | null | undefined, + items: readonly AgentJournalRenderItem[] ): StructuredAgentSessionTurnVerdict { - return evidence?.kind === 'exit-observed' - ? { state: 'interrupted', completedAt: evidence.observedAt } - : UNVERIFIABLE_TURN_VERDICT + if (!evidence) { + return UNVERIFIABLE_TURN_VERDICT + } + return { + state: 'interrupted', + completedAt: + evidence.kind === 'exit-observed' + ? evidence.observedAt + : lastLiveActivityAt(items, evidence.observedAt) + } +} + +/** A probe finds a dead child long after it died; the last thing the journal saw it do bounds its + * end, so the turn's duration never counts the time Orca itself was down. */ +function lastLiveActivityAt(items: readonly AgentJournalRenderItem[], probedAt: number): number { + let latest: number | null = null + for (const item of items) { + if ( + !item.recovered && + item.observedAt <= probedAt && + (latest === null || item.observedAt > latest) + ) { + latest = item.observedAt + } + } + return latest ?? probedAt } /** Revises every still-running lifecycle item in place, keeping its identity and start. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 0778d2711291..f0a42439a291 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -36,6 +36,7 @@ import type { StructuredAgentSessionAdapter } from './structured-agent-session-a import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { UNEXPECTED_PROVIDER_EXIT_OUTCOME } from './structured-agent-session-dead-generation-settlement' import { StructuredAgentSessionHost } from './structured-agent-session-host' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { @@ -217,6 +218,17 @@ function turnLifecycle(turnId: string) { return item ? { ...readAgentJournalTurn(item.body), recovered: item.recovered } : null } +/** When the journal last saw the running turn live; a probe-proven death ends the turn there. */ +function seededTurnSeenAt(): number { + const item = restoredJournal() + .snapshot() + .items.find((candidate) => readAgentJournalTurn(candidate.body)?.turnId === 'turn-1') + if (!item) { + throw new Error('expected the seeded turn') + } + return item.observedAt +} + function restoredJournal(): AgentSessionJournal { const restored = ( host as unknown as { sessions: Map } @@ -301,6 +313,16 @@ describe('already-wedged profiles become usable on load', () => { 'a quit that left the owner for a probe to prove gone', wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }), null, + { state: 'interrupted', completedAt: 'when last seen live' } + ], + [ + 'a quit that left an owner on a host this one cannot probe', + wedgedRecord({ + claimStatus: 'live', + handoffStage: null, + ownerProcess: { ...DEAD_OWNER, hostId: 'remote-host' } + }), + null, { state: 'unverifiable' } ] ] as const)( @@ -309,8 +331,18 @@ describe('already-wedged profiles become usable on load', () => { await seedStore({ ...seeded, lease: { ...seeded.lease, deathEvidence } }) await seedRunningTurn() const published: AgentSessionStatusSummary[] = [] + const remote = seeded.lease.ownerProcess?.hostId === 'remote-host' openHost({ - statusSink: { publish: (summary) => published.push(summary), forget: () => {} } + statusSink: { publish: (summary) => published.push(summary), forget: () => {} }, + // Loss of contact is never proof of death: a remote owner only ever probes indeterminate. + ...(remote + ? { + probeOwner: async () => ({ + outcome: 'indeterminate' as const, + reason: 'owner runs on remote-host, which this host cannot probe' + }) + } + : {}) }) await host.restoreReadableSessions() @@ -320,12 +352,33 @@ describe('already-wedged profiles become usable on load', () => { turnId: 'turn-1', startedAt: NOW - 5_000, recovered: true, - ...verdict + ...verdict, + ...('completedAt' in verdict && verdict.completedAt === 'when last seen live' + ? { completedAt: seededTurnSeenAt() } + : {}) }) expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) + // The provider's own reason reaches the chat only from a watched exit; a probe's is Orca's. + const statusRows = restoredJournal() + .snapshot() + .items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) + expect(statusRows).toEqual( + remote + ? [] + : [ + deathEvidence + ? 'The provider stopped while this response was in progress: provider exited: transport closed. You can continue in this conversation.' + : UNEXPECTED_PROVIDER_EXIT_OUTCOME + ] + ) // What the sidebar reads: every status this restart published says the chat is not working. expect(published.filter((summary) => summary.sessionId === SESSION)).not.toEqual([]) expect(published.map((summary) => summary.status)).not.toContain('working') + // A crash is not something the user did: no outcome is claimed, so no reader files it as a + // cancellation the user already knows about. + expect(published.map((summary) => summary.turnOutcome)).toEqual( + published.map(() => undefined) + ) } ) @@ -340,11 +393,12 @@ describe('already-wedged profiles become usable on load', () => { expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) - // A pid probe proved the owner gone; nobody saw it exit, so the turn has no end. + // A pid probe proved the owner gone, so the turn was cut short when it was last seen working. expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', - state: 'unverifiable', + state: 'interrupted', startedAt: NOW - 5_000, + completedAt: seededTurnSeenAt(), recovered: true }) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -435,11 +489,12 @@ describe('already-wedged profiles become usable on load', () => { expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' }) - // A pid probe proved the owner gone; nobody saw it exit, so the turn has no end. + // A pid probe proved the owner gone, so the turn was cut short when it was last seen working. expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', - state: 'unverifiable', + state: 'interrupted', startedAt: NOW - 5_000, + completedAt: seededTurnSeenAt(), recovered: true }) } From 9c46353cc28a8dd4acc8a7b1be24c469edb967f2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:31:33 -0700 Subject: [PATCH 02/24] fix(native-chat): end a probe-proven turn at the last row the journal wrote live A revised item keeps its first sighting's timestamp, so a long command or a streamed reply read as ending when it started. The reducer now tracks the latest live row the same way it tracks all activity. --- .../agent-session-journal/journal-reducer.ts | 4 + .../agent-session-journal/journal-store.ts | 3 + ...gent-session-dead-generation-settlement.ts | 2 +- ...d-agent-session-stale-turn-verdict.test.ts | 112 ++++++++++++++---- ...ctured-agent-session-stale-turn-verdict.ts | 24 +--- ...t-session-wedged-profile-migration.test.ts | 31 ++--- 6 files changed, 117 insertions(+), 59 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 273502c8cf2b..d43daba4c2b4 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -35,6 +35,8 @@ export type JournalReducerState = { epoch: string lastSequence: number lastActivityAt: number + /** `lastActivityAt` over rows written live: crash reconciliation is not the provider working. */ + lastLiveActivityAt: number /** Lowest sequence still individually replayable; rows below it were compacted. */ oldestSequence: number highestFence: number @@ -55,6 +57,7 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou epoch, lastSequence: 0, lastActivityAt: 0, + lastLiveActivityAt: 0, oldestSequence: 1, highestFence: 0, items: new Map(), @@ -73,6 +76,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } state.lastActivityAt = Math.max(state.lastActivityAt, row.ts) + state.lastLiveActivityAt = Math.max(state.lastLiveActivityAt, row.recovered ? 0 : row.ts) if (row.kind === 'item') { if (journalItemRevisionIsStale(state, row.itemId, row.revision)) { return diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 568fbe4d39d3..8679a538612c 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -220,6 +220,9 @@ export class AgentSessionJournal { /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt + /** The same over rows written live, so crash reconciliation never reads as activity; 0 if none. */ + lastLiveActivityAt = (): number => this.state.lastLiveActivityAt + submissions = (): AgentJournalSubmission[] => [...this.state.submissions.values()] pendingSubmissions = (): AgentJournalSubmission[] => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 6a7a2725ea53..66b492e4297f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -219,7 +219,7 @@ export async function settleStaleStructuredAgentSessionState(input: { }): Promise { const { journal } = input const items = journal.snapshot().items - const verdict = turnVerdictFromDeathEvidence(input.deathEvidence, items) + const verdict = turnVerdictFromDeathEvidence(input.deathEvidence, journal.lastLiveActivityAt()) const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` const mutations: JournalLifecycleMutationInput[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 86ba7b5dab47..2aa9c961349f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -4,9 +4,13 @@ import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { + settleStaleStructuredAgentSessionState, + UNEXPECTED_PROVIDER_EXIT_OUTCOME +} from './structured-agent-session-dead-generation-settlement' import { runningTurnLifecycleRevisions, turnVerdictFromDeathEvidence, @@ -75,39 +79,31 @@ function promptItem(state: 'pending' | 'resolved', sequence: number): AgentJourn } describe('turn verdict from death evidence', () => { - // A running turn first seen at 100, a tool call at 300, and a row crash recovery wrote at 450. - const items: AgentJournalRenderItem[] = [ - lifecycleItem('turn-2', 'running', 100), - { ...lifecycleItem('turn-tool', 'running', 300) }, - { ...lifecycleItem('turn-recovered', 'completed', 450), recovered: true, recoveredAt: 450 } - ] - it('ends a watched exit at the exit', () => { expect( - turnVerdictFromDeathEvidence( - { kind: 'exit-observed', detail: 'exit', observedAt: 500 }, - items - ) + turnVerdictFromDeathEvidence({ kind: 'exit-observed', detail: 'exit', observedAt: 500 }, 300) ).toEqual({ state: 'interrupted', completedAt: 500 }) }) it.each(['pid-absent', 'identity-mismatch'] as const)( - 'ends a %s proof at the last thing the journal saw live, not at the probe', + 'ends a %s proof at the last row the journal saw live, not at the probe', (kind) => { // Probed at 9000, long after the crash: the downtime is never counted as work. expect( - turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, items) + turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, 300) ).toEqual({ state: 'interrupted', completedAt: 300 }) - // Nothing live before the probe: the probe is the only bound there is. - expect(turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 50 }, items)).toEqual( - { state: 'interrupted', completedAt: 50 } - ) + // A live row stamped after the probe cannot outlast it, and no live row leaves only the probe. + for (const lastLive of [9_500, 0]) { + expect( + turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, lastLive) + ).toEqual({ state: 'interrupted', completedAt: 9_000 }) + } } ) it('leaves a release nothing proved unverifiable', () => { - expect(turnVerdictFromDeathEvidence(null, items)).toEqual({ state: 'unverifiable' }) - expect(turnVerdictFromDeathEvidence(undefined, items)).toEqual({ state: 'unverifiable' }) + expect(turnVerdictFromDeathEvidence(null, 300)).toEqual({ state: 'unverifiable' }) + expect(turnVerdictFromDeathEvidence(undefined, 300)).toEqual({ state: 'unverifiable' }) }) }) @@ -208,6 +204,7 @@ describe('stale session state on a cold acquire', () => { const journal = { snapshot: () => ({ items }), cursor: () => ({ epoch: 'epoch-1', sequence: 8 }), + lastLiveActivityAt: () => 2, appendLifecycleBatch } as unknown as AgentSessionJournal return { journal, appendLifecycleBatch } @@ -332,6 +329,81 @@ describe('stale session state on a cold acquire', () => { } }) + it('ends a probe-proven turn at its last live row, which a revised item does not carry', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) + const journals = createTrackedJournalOpener() + let now = 100 + try { + const journal = await journals.open({ + identity: { + sessionId: 'session-1', + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: THREAD } + }, + journalDir: root, + now: () => now + }) + const command = { provider: 'codex' as const, threadId: THREAD, turnId: 'turn-1', ordinal: 1 } + const shell = { kind: 'tool-call' as const, name: 'shell', input: { command: 'pnpm test' } } + await journal.appendItem( + { ...command, ordinal: 0 }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: 100 }, + { fence: 1 } + ) + now = 200 + await journal.appendItem(command, { ...shell, state: 'running' }, { fence: 1 }) + // Output streamed until 700: a revision, so the item still reads as first seen at 200. + now = 700 + await journal.appendItem( + command, + { ...shell, input: { command: 'pnpm test', streamed: 'ok' }, state: 'running' }, + { fence: 1 } + ) + // Crash reconciliation is Orca writing, not the provider working. + now = 900 + await journal.appendLifecycleBatch({ + settlementId: 'earlier-recovery', + fence: 1, + recovered: true, + mutations: [ + { + kind: 'item', + identity: { provider: 'orca', clientMessageId: 'earlier-recovery' }, + body: { kind: 'status', text: 'recovered' } + } + ] + }) + now = 9_000 + + await settleStaleStructuredAgentSessionState({ + journal, + sessionId: 'session-1', + fence: 2, + acquisitionGeneration: 'generation-2', + deathEvidence: { + kind: 'pid-absent', + detail: 'recorded pid absent on host', + observedAt: 9_000 + } + }) + + const items = journal.snapshot().items + expect(items.map((item) => readAgentJournalTurn(item.body)).find(Boolean)).toMatchObject({ + state: 'interrupted', + completedAt: 700 + }) + // The probe's detail is Orca's, so the row carries none. + expect( + items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) + ).toEqual(['recovered', UNEXPECTED_PROVIDER_EXIT_OUTCOME]) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + }) + it('writes nothing when no turn is running and keys on the journal position without a generation', async () => { const idle = journalWith([ lifecycleItem('turn-1', 'completed', 1, { startedAt: 10, completedAt: 20 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 30707cef6152..e851b1b57b26 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -27,36 +27,22 @@ export const UNVERIFIABLE_TURN_VERDICT: StructuredAgentSessionTurnVerdict = { export function turnVerdictFromDeathEvidence( evidence: AgentSessionDeathEvidence | null | undefined, - items: readonly AgentJournalRenderItem[] + lastLiveActivityAt: number ): StructuredAgentSessionTurnVerdict { if (!evidence) { return UNVERIFIABLE_TURN_VERDICT } + // A probe finds a dead child long after it died; the last row it wrote bounds its end, so the + // turn never counts the time Orca itself was down. return { state: 'interrupted', completedAt: - evidence.kind === 'exit-observed' + evidence.kind === 'exit-observed' || lastLiveActivityAt <= 0 ? evidence.observedAt - : lastLiveActivityAt(items, evidence.observedAt) + : Math.min(lastLiveActivityAt, evidence.observedAt) } } -/** A probe finds a dead child long after it died; the last thing the journal saw it do bounds its - * end, so the turn's duration never counts the time Orca itself was down. */ -function lastLiveActivityAt(items: readonly AgentJournalRenderItem[], probedAt: number): number { - let latest: number | null = null - for (const item of items) { - if ( - !item.recovered && - item.observedAt <= probedAt && - (latest === null || item.observedAt > latest) - ) { - latest = item.observedAt - } - } - return latest ?? probedAt -} - /** Revises every still-running lifecycle item in place, keeping its identity and start. */ export function runningTurnLifecycleRevisions( items: readonly AgentJournalRenderItem[], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index f0a42439a291..c3163a90e92e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -187,6 +187,9 @@ function isAcquirable(lease: NonNullable>['le ) } +/** When the seeded turn's row was written: the last time the dead owner was seen working. */ +const SEEDED_TURN_SEEN_AT = NOW - 4_000 + async function seedRunningTurn(provider: 'codex' | 'claude' = 'codex'): Promise { const journal = await openAgentSessionJournal({ identity: { @@ -199,7 +202,11 @@ async function seedRunningTurn(provider: 'codex' | 'claude' = 'codex'): Promise< ? { kind: 'codex', threadId: THREAD } : { kind: 'claude', sessionId: 'provider-session-alpha-1', leafUuid: null } }, - journalDir: journalDirectoryFor(root, { workspaceId: LOCATION.workspaceId, sessionId: SESSION }) + journalDir: journalDirectoryFor(root, { + workspaceId: LOCATION.workspaceId, + sessionId: SESSION + }), + now: () => SEEDED_TURN_SEEN_AT }) await journal.appendItem( provider === 'codex' @@ -218,17 +225,6 @@ function turnLifecycle(turnId: string) { return item ? { ...readAgentJournalTurn(item.body), recovered: item.recovered } : null } -/** When the journal last saw the running turn live; a probe-proven death ends the turn there. */ -function seededTurnSeenAt(): number { - const item = restoredJournal() - .snapshot() - .items.find((candidate) => readAgentJournalTurn(candidate.body)?.turnId === 'turn-1') - if (!item) { - throw new Error('expected the seeded turn') - } - return item.observedAt -} - function restoredJournal(): AgentSessionJournal { const restored = ( host as unknown as { sessions: Map } @@ -313,7 +309,7 @@ describe('already-wedged profiles become usable on load', () => { 'a quit that left the owner for a probe to prove gone', wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }), null, - { state: 'interrupted', completedAt: 'when last seen live' } + { state: 'interrupted', completedAt: SEEDED_TURN_SEEN_AT } ], [ 'a quit that left an owner on a host this one cannot probe', @@ -352,10 +348,7 @@ describe('already-wedged profiles become usable on load', () => { turnId: 'turn-1', startedAt: NOW - 5_000, recovered: true, - ...verdict, - ...('completedAt' in verdict && verdict.completedAt === 'when last seen live' - ? { completedAt: seededTurnSeenAt() } - : {}) + ...verdict }) expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) // The provider's own reason reaches the chat only from a watched exit; a probe's is Orca's. @@ -398,7 +391,7 @@ describe('already-wedged profiles become usable on load', () => { turnId: 'turn-1', state: 'interrupted', startedAt: NOW - 5_000, - completedAt: seededTurnSeenAt(), + completedAt: SEEDED_TURN_SEEN_AT, recovered: true }) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -494,7 +487,7 @@ describe('already-wedged profiles become usable on load', () => { turnId: 'turn-1', state: 'interrupted', startedAt: NOW - 5_000, - completedAt: seededTurnSeenAt(), + completedAt: SEEDED_TURN_SEEN_AT, recovered: true }) } From 498d53920a1108e88b3c42057ff91a0db6baa96e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:39:09 -0700 Subject: [PATCH 03/24] test(native-chat): give the unexpected-exit fake journal its live-activity read --- .../structured-agent-session-unexpected-exit.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 98901380aa39..1b0670a15d92 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -129,6 +129,7 @@ describe('provider-exit recovery tickets', () => { snapshot: () => ({ items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })] }), + lastLiveActivityAt: () => 1_000, appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []) } From f82f3edfe0cf40b6b72a2e2fbc56b9f1882e44be Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:43:14 -0700 Subject: [PATCH 04/24] refactor(native-chat): read the journal's live bound only for a probe-proven death --- ...gent-session-dead-generation-settlement.ts | 2 +- ...d-agent-session-stale-turn-verdict.test.ts | 21 +++++++++++++------ ...ctured-agent-session-stale-turn-verdict.ts | 12 ++++++----- ...ured-agent-session-unexpected-exit.test.ts | 1 - 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 66b492e4297f..8c1f8e8196fa 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -219,7 +219,7 @@ export async function settleStaleStructuredAgentSessionState(input: { }): Promise { const { journal } = input const items = journal.snapshot().items - const verdict = turnVerdictFromDeathEvidence(input.deathEvidence, journal.lastLiveActivityAt()) + const verdict = turnVerdictFromDeathEvidence(input.deathEvidence, journal) const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` const mutations: JournalLifecycleMutationInput[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 2aa9c961349f..3abff16317b2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -79,9 +79,14 @@ function promptItem(state: 'pending' | 'resolved', sequence: number): AgentJourn } describe('turn verdict from death evidence', () => { + const lastLiveAt = (at: number) => ({ lastLiveActivityAt: () => at }) + it('ends a watched exit at the exit', () => { expect( - turnVerdictFromDeathEvidence({ kind: 'exit-observed', detail: 'exit', observedAt: 500 }, 300) + turnVerdictFromDeathEvidence( + { kind: 'exit-observed', detail: 'exit', observedAt: 500 }, + lastLiveAt(300) + ) ).toEqual({ state: 'interrupted', completedAt: 500 }) }) @@ -90,20 +95,25 @@ describe('turn verdict from death evidence', () => { (kind) => { // Probed at 9000, long after the crash: the downtime is never counted as work. expect( - turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, 300) + turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, lastLiveAt(300)) ).toEqual({ state: 'interrupted', completedAt: 300 }) // A live row stamped after the probe cannot outlast it, and no live row leaves only the probe. for (const lastLive of [9_500, 0]) { expect( - turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, lastLive) + turnVerdictFromDeathEvidence( + { kind, detail: 'gone', observedAt: 9_000 }, + lastLiveAt(lastLive) + ) ).toEqual({ state: 'interrupted', completedAt: 9_000 }) } } ) it('leaves a release nothing proved unverifiable', () => { - expect(turnVerdictFromDeathEvidence(null, 300)).toEqual({ state: 'unverifiable' }) - expect(turnVerdictFromDeathEvidence(undefined, 300)).toEqual({ state: 'unverifiable' }) + expect(turnVerdictFromDeathEvidence(null, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) + expect(turnVerdictFromDeathEvidence(undefined, lastLiveAt(300))).toEqual({ + state: 'unverifiable' + }) }) }) @@ -204,7 +214,6 @@ describe('stale session state on a cold acquire', () => { const journal = { snapshot: () => ({ items }), cursor: () => ({ epoch: 'epoch-1', sequence: 8 }), - lastLiveActivityAt: () => 2, appendLifecycleBatch } as unknown as AgentSessionJournal return { journal, appendLifecycleBatch } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index e851b1b57b26..bfc73ef22c2a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -16,6 +16,7 @@ import { } from '../../../shared/agent-session-turn-record' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' export type StructuredAgentSessionTurnVerdict = | { state: 'interrupted'; completedAt: number } @@ -27,19 +28,20 @@ export const UNVERIFIABLE_TURN_VERDICT: StructuredAgentSessionTurnVerdict = { export function turnVerdictFromDeathEvidence( evidence: AgentSessionDeathEvidence | null | undefined, - lastLiveActivityAt: number + journal: Pick ): StructuredAgentSessionTurnVerdict { if (!evidence) { return UNVERIFIABLE_TURN_VERDICT } + if (evidence.kind === 'exit-observed') { + return { state: 'interrupted', completedAt: evidence.observedAt } + } // A probe finds a dead child long after it died; the last row it wrote bounds its end, so the // turn never counts the time Orca itself was down. + const lastLive = journal.lastLiveActivityAt() return { state: 'interrupted', - completedAt: - evidence.kind === 'exit-observed' || lastLiveActivityAt <= 0 - ? evidence.observedAt - : Math.min(lastLiveActivityAt, evidence.observedAt) + completedAt: lastLive > 0 ? Math.min(lastLive, evidence.observedAt) : evidence.observedAt } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 1b0670a15d92..98901380aa39 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -129,7 +129,6 @@ describe('provider-exit recovery tickets', () => { snapshot: () => ({ items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })] }), - lastLiveActivityAt: () => 1_000, appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []) } From 25e6f518d6b43ac15e860fce2ba9ec771e7d4651 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 13:49:39 -0700 Subject: [PATCH 05/24] fix(native-chat): mark what a journal open settles for a gone host as crash reconciliation A crashed host's working roster is retired when the journal reopens. That row was written live, so a probe-proven turn ended at the relaunch and counted the downtime. --- .../journal-store-restore.ts | 4 +- ...d-agent-session-stale-turn-verdict.test.ts | 56 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index fc69dd339d8b..ffa63e436a83 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -39,7 +39,9 @@ export function restoreJournalStore( publishRepairEpoch: () => collaborators.epochController.start('unreconcilable_prefix', host.state().highestFence), adopt: host.adopt, - appendItem: (identity, body, fence) => host.journal().appendItem(identity, body, { fence }), + // What an open says about a host that is gone is crash reconciliation, never the provider working. + appendItem: (identity, body, fence) => + host.journal().appendItem(identity, body, { fence, recovered: true }), agent: host.identity.agent, highestFence: () => host.state().highestFence, malformedRows: host.malformedRows, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 3abff16317b2..97a8b887aa8b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -5,6 +5,10 @@ import { describe, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + codexSubagentGroupBody, + codexSubagentGroupIdentity +} from '../../codex/codex-subagent-roster' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import { @@ -413,6 +417,58 @@ describe('stale session state on a cold acquire', () => { } }) + it('does not count the reopen settling a roster the crashed host left working as activity', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) + const journals = createTrackedJournalOpener() + let now = 100 + const identity = { + sessionId: 'session-1', + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex' as const, + providerHandle: { kind: 'codex' as const, threadId: THREAD } + } + try { + const live = await journals.open({ identity, journalDir: root, now: () => now }) + await live.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 0 }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: 100 }, + { fence: 1 } + ) + now = 200 + const groupId = `${THREAD}:turn-1` + await live.appendItem( + codexSubagentGroupIdentity(groupId), + codexSubagentGroupBody(groupId, [ + { id: 'a', label: 'explore', state: 'working', startedAt: 200 } + ]), + { fence: 1 } + ) + await live.close() + // Relaunched long after the crash: the open retires the roster, then the probe proves death. + now = 9_000 + const reopened = await journals.open({ identity, journalDir: root, now: () => now }) + + await settleStaleStructuredAgentSessionState({ + journal: reopened, + sessionId: 'session-1', + fence: 2, + acquisitionGeneration: 'generation-2', + deathEvidence: { kind: 'pid-absent', detail: 'gone', observedAt: 8_500 } + }) + + expect( + reopened + .snapshot() + .items.map((item) => readAgentJournalTurn(item.body)) + .find(Boolean) + ).toMatchObject({ state: 'interrupted', completedAt: 200 }) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + }) + it('writes nothing when no turn is running and keys on the journal position without a generation', async () => { const idle = journalWith([ lifecycleItem('turn-1', 'completed', 1, { startedAt: 10, completedAt: 20 }) From 707635179110a7e607e1b0f17c2df79877dc63d8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:16:45 -0700 Subject: [PATCH 06/24] fix(native-chat): bound a probe-proven death with the last time Orca proved the owner alive A crash mid-tool left the turn ending at the tool call's start, because Claude writes nothing while a Bash call runs. The death evidence now records the lease's last renewal before the death as lastProvenAliveAt, and the turn ends at the later of that and the last live row, capped at the probe. Parking a lease in recovery no longer stamps lastRenewedAt, since nothing proved the owner alive then; a child that outlived Orca would otherwise have its turn count the downtime. --- ...tured-agent-session-crash-turn-end.test.ts | 211 ++++++++++++++++++ ...d-agent-session-stale-turn-verdict.test.ts | 39 +++- ...ctured-agent-session-stale-turn-verdict.ts | 8 +- ...-session-acquisition-failure-settlement.ts | 33 +-- ...session-death-evidence-persistence.test.ts | 89 ++++++++ ...agent-session-restart-lease-transitions.ts | 15 +- .../agent-session-lease-adjudication.test.ts | 22 +- .../agent-session-lease-adjudication.ts | 13 +- src/shared/agent-session-record.ts | 15 +- 9 files changed, 411 insertions(+), 34 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts create mode 100644 src/main/runtime/agent-session-death-evidence-persistence.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts new file mode 100644 index 000000000000..1e8d047145e8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -0,0 +1,211 @@ +// Where a turn a crash cut short ends, when the provider wrote nothing while it worked. +// +// Claude reports a Bash call once when it starts and again only when it finishes, so a command +// that runs for half a minute leaves one journal row at its start. The lease renewal the host +// wrote every ten seconds is what saw the child working after that row. + +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { completedStructuredAgentTurnSeconds } from '../../../shared/structured-agent-session-turn-timing' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { AGENT_SESSION_STORE_FILE_NAME } from '../../runtime/agent-session-record-store-file' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { + HOST_TEST_LOCATION as LOCATION, + HOST_TEST_SESSION as SESSION +} from './structured-agent-session-host-test-data' + +const PROVIDER_SESSION = 'provider-session-alpha-1' +/** The tool call's row: the last thing the provider wrote before the crash. */ +const TOOL_STARTED_AT = 1_800_000_000_000 +/** The last renewal before the crash, while the command was still running. */ +const LAST_RENEWED_AT = TOOL_STARTED_AT + 25_000 +/** Orca comes back an hour later. */ +const RELAUNCHED_AT = TOOL_STARTED_AT + 60 * 60 * 1000 + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost + +function crashedClaudeRecord(): AgentSessionRecord { + const linkId = 'claude-13-link' + return { + schemaVersion: 2, + sessionId: SESSION, + location: LOCATION, + provider: 'claude', + providerHandleChain: [ + { + linkId, + handle: { provider: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null }, + origin: 'created', + mintedAtFence: 13, + observedAt: TOOL_STARTED_AT - 60_000 + } + ], + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + createdAt: TOOL_STARTED_AT - 60_000, + updatedAt: LAST_RENEWED_AT, + lease: { + sessionId: SESSION, + runtimeKind: 'native', + runtimeFence: 13, + handoffStage: null, + provenHandleLinkId: linkId, + ownerProcess: { + hostId: 'local', + pid: 12_546, + processStartTimeMs: TOOL_STARTED_AT - 60_000, + spawnToken: 'spawn-crashed' + }, + reservedSpawnToken: 'spawn-crashed', + leaseDeadlineAt: LAST_RENEWED_AT + 30_000, + lastRenewedAt: LAST_RENEWED_AT, + handoffOperationId: null, + journalCheckpoint: null, + claimKeyId: 'key-1', + claimStatus: 'live', + unreconciled: false, + deathEvidence: null + } + } +} + +async function seedCrashedStore(): Promise { + const directory = join(root, 'store') + await mkdir(directory, { recursive: true }) + await writeFile( + join(directory, AGENT_SESSION_STORE_FILE_NAME), + JSON.stringify({ + schemaVersion: 2, + hostId: 'local', + records: { [SESSION]: crashedClaudeRecord() }, + operations: {}, + retiredClaimKeys: [], + unusableRecords: {} + }), + 'utf-8' + ) + store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) +} + +/** A running turn whose only row after its start is a Bash call that never reported back. */ +async function seedClaudeToolTurn(): Promise { + let now = TOOL_STARTED_AT - 2_000 + const journal = await openAgentSessionJournal({ + identity: { + sessionId: SESSION, + workspaceId: LOCATION.workspaceId, + hostId: LOCATION.executionHostId, + agent: 'claude', + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null } + }, + journalDir: journalDirectoryFor(root, { + workspaceId: LOCATION.workspaceId, + sessionId: SESSION + }), + now: () => now + }) + await journal.appendItem( + { provider: 'claude', sessionId: PROVIDER_SESSION, uuid: 'uuid-turn' }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: now }, + { fence: 13 } + ) + now = TOOL_STARTED_AT + await journal.appendItem( + { provider: 'claude', sessionId: PROVIDER_SESSION, uuid: 'uuid-bash' }, + { + kind: 'tool-call', + name: 'Bash', + input: { command: 'for i in $(seq 90); do sleep 1; done' }, + state: 'running' + }, + { fence: 13 } + ) + await journal.close() +} + +function openHost(overrides: Partial): void { + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire: vi.fn(), + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-new', + now: () => RELAUNCHED_AT, + ...overrides + }) +} + +/** What a client reads over the wire. */ +function settledTurn() { + return host + .journalSnapshot(SESSION) + .items.map((item) => readAgentJournalTurn(item.body)) + .find(Boolean) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-crash-turn-end-')) + await seedCrashedStore() + await seedClaudeToolTurn() +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +describe('a turn a crash cut short mid-tool', () => { + it('ends at the last renewal, not at the tool call the provider last reported', async () => { + openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) + + await host.restoreReadableSessions() + + expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ + kind: 'pid-absent', + observedAt: RELAUNCHED_AT, + lastProvenAliveAt: LAST_RENEWED_AT + }) + expect(settledTurn()).toMatchObject({ state: 'interrupted', completedAt: LAST_RENEWED_AT }) + // "Worked for 27s", where the tool call's row alone reads 2s. + expect(completedStructuredAgentTurnSeconds(settledTurn())).toBe(27) + }) + + it('ends at the pre-crash renewal when the child outlived Orca and recovery stopped it', async () => { + // The orphan is alive at relaunch, but its output went nowhere: none of that is work shown. + let alive = true + const probeOwner = async (): Promise => + alive + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + openHost({ probeOwner, stopOwnerProcess }) + + await host.restoreReadableSessions() + + expect(stopOwnerProcess).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ + kind: 'pid-absent', + lastProvenAliveAt: LAST_RENEWED_AT + }) + expect(settledTurn()).toMatchObject({ state: 'interrupted', completedAt: LAST_RENEWED_AT }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 97a8b887aa8b..59b13c8c5798 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -95,7 +95,7 @@ describe('turn verdict from death evidence', () => { }) it.each(['pid-absent', 'identity-mismatch'] as const)( - 'ends a %s proof at the last row the journal saw live, not at the probe', + 'ends a %s proof an older build recorded at the last row the journal saw live, not at the probe', (kind) => { // Probed at 9000, long after the crash: the downtime is never counted as work. expect( @@ -113,6 +113,43 @@ describe('turn verdict from death evidence', () => { } ) + it('ends a probe-proven death at the later of the last renewal and the last live row', () => { + const proof = (lastProvenAliveAt: number) => ({ + kind: 'pid-absent' as const, + detail: 'gone', + observedAt: 9_000, + lastProvenAliveAt + }) + // A silent tool run: the renewal saw the child working long after its last row. + expect(turnVerdictFromDeathEvidence(proof(8_000), lastLiveAt(300))).toEqual({ + state: 'interrupted', + completedAt: 8_000 + }) + expect(turnVerdictFromDeathEvidence(proof(8_000), lastLiveAt(0))).toEqual({ + state: 'interrupted', + completedAt: 8_000 + }) + // A chatty provider: its last row is the tighter bound. + expect(turnVerdictFromDeathEvidence(proof(200), lastLiveAt(300))).toEqual({ + state: 'interrupted', + completedAt: 300 + }) + // Neither bound outlasts the probe. + expect(turnVerdictFromDeathEvidence(proof(9_500), lastLiveAt(300))).toEqual({ + state: 'interrupted', + completedAt: 9_000 + }) + }) + + it('ends a watched exit at the exit even when a renewal is recorded', () => { + expect( + turnVerdictFromDeathEvidence( + { kind: 'exit-observed', detail: 'exit', observedAt: 500, lastProvenAliveAt: 400 }, + lastLiveAt(300) + ) + ).toEqual({ state: 'interrupted', completedAt: 500 }) + }) + it('leaves a release nothing proved unverifiable', () => { expect(turnVerdictFromDeathEvidence(null, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) expect(turnVerdictFromDeathEvidence(undefined, lastLiveAt(300))).toEqual({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index bfc73ef22c2a..4d0c502a70db 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -36,12 +36,12 @@ export function turnVerdictFromDeathEvidence( if (evidence.kind === 'exit-observed') { return { state: 'interrupted', completedAt: evidence.observedAt } } - // A probe finds a dead child long after it died; the last row it wrote bounds its end, so the - // turn never counts the time Orca itself was down. - const lastLive = journal.lastLiveActivityAt() + // A probe finds a dead child long after it died. The later of the last renewal and the last row + // it wrote bounds its end, so the turn never counts the time Orca itself was down. + const lastSeen = Math.max(evidence.lastProvenAliveAt ?? 0, journal.lastLiveActivityAt()) return { state: 'interrupted', - completedAt: lastLive > 0 ? Math.min(lastLive, evidence.observedAt) : evidence.observedAt + completedAt: lastSeen > 0 ? Math.min(lastSeen, evidence.observedAt) : evidence.observedAt } } diff --git a/src/main/runtime/agent-session-acquisition-failure-settlement.ts b/src/main/runtime/agent-session-acquisition-failure-settlement.ts index a3e114081186..2b28adc8b825 100644 --- a/src/main/runtime/agent-session-acquisition-failure-settlement.ts +++ b/src/main/runtime/agent-session-acquisition-failure-settlement.ts @@ -86,12 +86,14 @@ export function settleFailedAgentSessionPostAcquisitionAttachment( } const next = args.exitProof === 'unproven' - ? withLease(record, { - ...record.lease, - handoffStage: 'recovering', - handoffOperationId: null, - lastRenewedAt: args.now - }) + ? { + ...withLease(record, { + ...record.lease, + handoffStage: 'recovering', + handoffOperationId: null + }), + updatedAt: args.now + } : withLease(record, { ...record.lease, runtimeFence: nextAgentSessionFence(record.lease), @@ -133,14 +135,17 @@ function settleFailedLease( throw new Error('agent_session_ownership_unknown') } if (args.exitProof === 'unproven' && record.lease.ownerProcess) { - return withLease(record, { - ...record.lease, - handoffStage: 'recovering', - // The operation is durably settled failed below; a lease still naming it would - // read as an in-flight transfer to every consumer that keys on the stage + id pair. - handoffOperationId: null, - lastRenewedAt: args.now - }) + // Parking in recovery proves nothing alive, so `lastRenewedAt` keeps the spawn's proof. + return { + ...withLease(record, { + ...record.lease, + handoffStage: 'recovering', + // The operation is durably settled failed below; a lease still naming it would + // read as an in-flight transfer to every consumer that keys on the stage + id pair. + handoffOperationId: null + }), + updatedAt: args.now + } } return withLease(record, { ...record.lease, diff --git a/src/main/runtime/agent-session-death-evidence-persistence.test.ts b/src/main/runtime/agent-session-death-evidence-persistence.test.ts new file mode 100644 index 000000000000..4ec7985c2038 --- /dev/null +++ b/src/main/runtime/agent-session-death-evidence-persistence.test.ts @@ -0,0 +1,89 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { AgentSessionDeathEvidence } from '../../shared/agent-session-record' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../shared/agent-session-record.test-fixture' +import { AgentSessionRecordStore } from './agent-session-record-store' +import { agentSessionStorePath } from './agent-session-record-store-file' + +const SESSION = 'session-alpha-1' +/** The fixture lease's last renewal. */ +const LAST_RENEWED_AT = 30_000 + +let directory: string + +async function seed(deathEvidence: AgentSessionDeathEvidence | null): Promise { + const lease = + deathEvidence === null + ? agentSessionLeaseFixture() + : agentSessionLeaseFixture({ + ownerProcess: null, + reservedSpawnToken: null, + claimStatus: 'released', + deathEvidence + }) + await writeFile( + agentSessionStorePath(directory), + JSON.stringify({ + schemaVersion: 2, + hostId: 'local', + records: { [SESSION]: agentSessionRecordFixture(lease) }, + operations: {}, + retiredClaimKeys: [], + unusableRecords: {} + }) + ) +} + +function open(): Promise { + return AgentSessionRecordStore.open({ directory, hostId: 'local' }) +} + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'orca-death-evidence-')) +}) + +afterEach(async () => { + await rm(directory, { recursive: true, force: true }) +}) + +describe('death evidence on disk', () => { + it('records the last renewal before the crash and reads it back after another restart', async () => { + await seed(null) + const crashed = await open() + await crashed.reconcileOnRestart({ + probe: async () => ({ outcome: 'pid-absent' }), + now: 90_000 + }) + const evidence = { + kind: 'pid-absent', + detail: 'recorded pid absent on host', + observedAt: 90_000, + lastProvenAliveAt: LAST_RENEWED_AT + } + expect(crashed.getRecord(SESSION)?.lease.deathEvidence).toEqual(evidence) + expect((await open()).getRecord(SESSION)?.lease.deathEvidence).toEqual(evidence) + }) + + it('loads evidence an older build wrote without a proven-alive time', async () => { + const olderBuild = { kind: 'pid-absent' as const, detail: 'gone', observedAt: 90_000 } + await seed(olderBuild) + const store = await open() + expect(store.isSessionUnreadable(SESSION)).toBe(false) + expect(store.getRecord(SESSION)?.lease.deathEvidence).toEqual(olderBuild) + }) + + it('quarantines evidence that proves the owner alive after the probe found it gone', async () => { + await seed({ + kind: 'pid-absent', + detail: 'gone', + observedAt: 90_000, + lastProvenAliveAt: 90_001 + }) + expect((await open()).isSessionUnreadable(SESSION)).toBe(true) + }) +}) diff --git a/src/main/runtime/agent-session-restart-lease-transitions.ts b/src/main/runtime/agent-session-restart-lease-transitions.ts index 70104588a584..b4269d21c677 100644 --- a/src/main/runtime/agent-session-restart-lease-transitions.ts +++ b/src/main/runtime/agent-session-restart-lease-transitions.ts @@ -52,10 +52,13 @@ export function applyAgentSessionRestartAdjudication(args: { deathEvidence: adjudication.evidence }) } - return withLease(record, { - ...record.lease, - handoffStage: adjudication.stage, - unreconciled: false, - lastRenewedAt: args.now - }) + // Parking in recovery proves nothing alive, so `lastRenewedAt` keeps the pre-crash proof. + return { + ...withLease(record, { + ...record.lease, + handoffStage: adjudication.stage, + unreconciled: false + }), + updatedAt: args.now + } } diff --git a/src/shared/agent-session-lease-adjudication.test.ts b/src/shared/agent-session-lease-adjudication.test.ts index 2216782b5e5c..851eda08281d 100644 --- a/src/shared/agent-session-lease-adjudication.test.ts +++ b/src/shared/agent-session-lease-adjudication.test.ts @@ -208,14 +208,27 @@ describe('restart reconciliation', () => { expect(result).toEqual({ disposition: 'evicted', nextFence: 8, + // The death happened between the last renewal and the probe. evidence: { kind: 'identity-mismatch', detail: 'mismatched process-start-time', - observedAt: 9_000 + observedAt: 9_000, + lastProvenAliveAt: 500 } }) }) + it('never records the owner alive after the probe that found it gone', () => { + // A clock stepped back across the restart would otherwise write an interval load rejects. + expect( + adjudicateAgentSessionRestart({ + lease: lease({ lastRenewedAt: 9_500 }), + probe: { outcome: 'pid-absent' }, + observedAt: 9_000 + }) + ).toMatchObject({ evidence: { observedAt: 9_000, lastProvenAliveAt: 9_000 } }) + }) + it('hands an unverifiable owner to recovery resolution instead of evicting it', () => { expect( adjudicateAgentSessionRestart({ @@ -255,7 +268,12 @@ describe('restart reconciliation', () => { ).toEqual({ disposition: 'evicted', nextFence: 8, - evidence: { kind: 'pid-absent', detail: 'recorded pid absent on host', observedAt: 9_000 } + evidence: { + kind: 'pid-absent', + detail: 'recorded pid absent on host', + observedAt: 9_000, + lastProvenAliveAt: 500 + } }) }) diff --git a/src/shared/agent-session-lease-adjudication.ts b/src/shared/agent-session-lease-adjudication.ts index ecdb1cecc3a7..d29d5f68d7ce 100644 --- a/src/shared/agent-session-lease-adjudication.ts +++ b/src/shared/agent-session-lease-adjudication.ts @@ -71,16 +71,19 @@ export function isProvenAliveProbe(probe: AgentSessionOwnerProbe): boolean { function deathEvidenceFor( probe: AgentSessionOwnerProbe, - observedAt: number + observedAt: number, + lease: AgentSessionLease ): AgentSessionDeathEvidence | null { + // Capped so a clock that stepped back across the restart still writes a valid interval. + const interval = { observedAt, lastProvenAliveAt: Math.min(lease.lastRenewedAt, observedAt) } if (probe.outcome === 'exit-observed') { - return { kind: 'exit-observed', detail: 'observed process exit', observedAt } + return { kind: 'exit-observed', detail: 'observed process exit', ...interval } } if (probe.outcome === 'pid-absent') { - return { kind: 'pid-absent', detail: 'recorded pid absent on host', observedAt } + return { kind: 'pid-absent', detail: 'recorded pid absent on host', ...interval } } if (probe.outcome === 'identity-mismatch') { - return { kind: 'identity-mismatch', detail: `mismatched ${probe.field}`, observedAt } + return { kind: 'identity-mismatch', detail: `mismatched ${probe.field}`, ...interval } } return null } @@ -219,7 +222,7 @@ export function adjudicateAgentSessionRestart(args: { reason: 'owner outlived the runtime that held its transport' } } - const evidence = deathEvidenceFor(probe, observedAt) + const evidence = deathEvidenceFor(probe, observedAt, lease) if (evidence) { return { disposition: 'evicted', nextFence: nextAgentSessionFence(lease), evidence } } diff --git a/src/shared/agent-session-record.ts b/src/shared/agent-session-record.ts index 2b85695a8459..bd0e166437a8 100644 --- a/src/shared/agent-session-record.ts +++ b/src/shared/agent-session-record.ts @@ -87,6 +87,9 @@ export type AgentSessionDeathEvidence = { kind: 'exit-observed' | 'pid-absent' | 'identity-mismatch' detail: string observedAt: number + /** The death interval's lower bound: the last time the runtime holding the owner's transport + * proved it alive. Absent when nothing did, and on evidence older builds wrote. */ + lastProvenAliveAt?: number } export type AgentSessionLease = { @@ -102,6 +105,8 @@ export type AgentSessionLease = { /** Reserved before any process exists, then matched against the child's environment. */ reservedSpawnToken: string | null leaseDeadlineAt: number + /** While `ownerProcess` is set, the last time its transport holder proved it alive; parking in + * `recovering` proves nothing, so it leaves this alone. */ lastRenewedAt: number handoffOperationId: string | null journalCheckpoint: AgentSessionJournalCheckpoint | null @@ -276,13 +281,19 @@ function isAgentSessionDeathEvidence(value: unknown): value is AgentSessionDeath return false } const evidence = value as Partial + const { observedAt, lastProvenAliveAt } = evidence return ( (evidence.kind === 'exit-observed' || evidence.kind === 'pid-absent' || evidence.kind === 'identity-mismatch') && isBoundedString(evidence.detail, MAX_ID_LENGTH) && - Number.isSafeInteger(evidence.observedAt) && - (evidence.observedAt as number) >= 0 + typeof observedAt === 'number' && + Number.isSafeInteger(observedAt) && + observedAt >= 0 && + (lastProvenAliveAt === undefined || + (Number.isSafeInteger(lastProvenAliveAt) && + lastProvenAliveAt >= 0 && + lastProvenAliveAt <= observedAt)) ) } From 577b9abe6301824539876bff8a0145fae83dceeb Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:20:13 -0700 Subject: [PATCH 07/24] test(native-chat): a failed acquisition parked in recovery keeps its last proof of life, and the timing read goes through the display selector --- ...tured-agent-session-crash-turn-end.test.ts | 13 ++- ...session-death-evidence-persistence.test.ts | 81 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index 1e8d047145e8..519ce1165fa9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -11,7 +11,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' -import { completedStructuredAgentTurnSeconds } from '../../../shared/structured-agent-session-turn-timing' +import { + completedStructuredAgentTurnSeconds, + selectStructuredAgentTurnTimings +} from '../../../shared/structured-agent-session-turn-timing' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { AGENT_SESSION_STORE_FILE_NAME } from '../../runtime/agent-session-record-store-file' import { journalDirectoryFor } from '../agent-session-journal/journal-paths' @@ -114,6 +117,11 @@ async function seedClaudeToolTurn(): Promise { }), now: () => now }) + await journal.appendItem( + { provider: 'claude', sessionId: PROVIDER_SESSION, uuid: 'uuid-user' }, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'run the loop' }] }, + { fence: 13 } + ) await journal.appendItem( { provider: 'claude', sessionId: PROVIDER_SESSION, uuid: 'uuid-turn' }, { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: now }, @@ -184,7 +192,8 @@ describe('a turn a crash cut short mid-tool', () => { }) expect(settledTurn()).toMatchObject({ state: 'interrupted', completedAt: LAST_RENEWED_AT }) // "Worked for 27s", where the tool call's row alone reads 2s. - expect(completedStructuredAgentTurnSeconds(settledTurn())).toBe(27) + const [timing] = selectStructuredAgentTurnTimings(host.journalSnapshot(SESSION).items).values() + expect(completedStructuredAgentTurnSeconds(timing)).toBe(27) }) it('ends at the pre-crash renewal when the child outlived Orca and recovery stopped it', async () => { diff --git a/src/main/runtime/agent-session-death-evidence-persistence.test.ts b/src/main/runtime/agent-session-death-evidence-persistence.test.ts index 4ec7985c2038..f00efa484a8b 100644 --- a/src/main/runtime/agent-session-death-evidence-persistence.test.ts +++ b/src/main/runtime/agent-session-death-evidence-persistence.test.ts @@ -7,6 +7,7 @@ import { agentSessionLeaseFixture, agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' +import type { AgentSessionFailedAcquisitionSettlement } from './agent-session-acquisition-failure-settlement' import { AgentSessionRecordStore } from './agent-session-record-store' import { agentSessionStorePath } from './agent-session-record-store-file' @@ -87,3 +88,83 @@ describe('death evidence on disk', () => { expect((await open()).isSessionUnreadable(SESSION)).toBe(true) }) }) + +describe('a failed acquisition parked in recovery', () => { + const NOW = 1_800_000_000_000 + const OPERATION_ID = `${NOW}-${'1'.padStart(32, '0')}` + + /** Reserve and observe the spawn at NOW, as an attach does before it can fail. */ + async function spawnedOwner(store: AgentSessionRecordStore): Promise { + const reserved = await store.reserveOwner({ + sessionId: SESSION, + location: agentSessionRecordFixture().location, + provider: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + expectedFence: null, + spawnToken: 'spawn-a', + claimKeyId: 'key-1', + handoffOperationId: OPERATION_ID, + probe: { outcome: 'indeterminate', reason: 'no answer' }, + operation: { callerKey: 'client-1', operationId: OPERATION_ID, fingerprint: 'fp-1' }, + now: NOW + }) + const fence = reserved.record.lease.runtimeFence + await store.commitProcessIdentity({ + sessionId: SESSION, + fence, + process: { hostId: 'local', pid: 4242, processStartTimeMs: NOW, spawnToken: 'spawn-a' }, + now: NOW + }) + return fence + } + + function unproven(fence: number, now: number): AgentSessionFailedAcquisitionSettlement { + return { + sessionId: SESSION, + fence, + spawnToken: 'spawn-a', + callerKey: 'client-1', + operationId: OPERATION_ID, + outcome: { status: 'failed', code: 'agent_session_operation_invalid', message: 'failed' }, + exitProof: 'unproven', + now + } + } + + it.each([ + ['before the owner proved its handle', false], + ['after the owner proved its handle', true] + ] as const)('keeps the last proof of life, not the failure, %s', async (_when, proved) => { + const store = await open() + const fence = await spawnedOwner(store) + if (proved) { + await store.proveOwner({ + sessionId: SESSION, + fence, + link: { + linkId: 'link-1', + handle: { provider: 'claude', sessionId: 'provider-session-1', leafUuid: null }, + origin: 'created', + mintedAtFence: fence, + observedAt: NOW + }, + now: NOW + }) + await store.settleFailedPostAcquisitionAttachment(unproven(fence, NOW + 60_000)) + } else { + await store.settleFailedAcquisition(unproven(fence, NOW + 60_000)) + } + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + handoffStage: 'recovering', + lastRenewedAt: NOW + }) + + const evicted = await store.evictProvenDeadOwner({ + sessionId: SESSION, + expectedFence: fence, + probe: { outcome: 'pid-absent' }, + now: NOW + 120_000 + }) + expect(evicted.lease.deathEvidence).toMatchObject({ lastProvenAliveAt: NOW }) + }) +}) From 65629a3f9f45e6d5cc43866efc448e8ba40aac57 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:10:36 -0700 Subject: [PATCH 08/24] refactor(native-chat): move the submission dispatch folds out of the journal reducer Main grew the reducer to its line limit, so the live-activity bound tipped it over. The dispatch row and echo-acceptance folds move unchanged into their own module. --- .../agent-session-journal/journal-reducer.ts | 80 ++----------------- .../journal-submission-dispatch-state.ts | 75 +++++++++++++++++ 2 files changed, 82 insertions(+), 73 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-submission-dispatch-state.ts diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index d363d9abd296..f5c9f539e779 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -25,6 +25,10 @@ import { } from '../../../shared/agent-session-journal-producer' import { structuredAgentSessionPayloadFingerprint } from '../../../shared/structured-agent-session-mutation' import { journalItemRevisionIsStale } from './journal-item-revision' +import { + acceptJournalSubmissionFromProviderItem, + applyJournalDispatchRow +} from './journal-submission-dispatch-state' import type { JournalRow } from './journal-row-schema' import { dispatchRejectionWasTransportWriteFailure } from '../../../shared/structured-agent-session-dispatch-rejection' @@ -82,7 +86,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } const itemId = resolveJournalItemId(state, row.itemId, row.body) - acceptSubmissionFromProviderItem(state, row.itemId, itemId, row) + acceptJournalSubmissionFromProviderItem(state, row.itemId, itemId, row) upsertItem(state, itemId, row.revision, journalRenderItem(itemId, row.revision, row.body, row)) return } @@ -100,7 +104,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo continue } const itemId = resolveJournalItemId(state, mutation.itemId, mutation.body) - acceptSubmissionFromProviderItem(state, mutation.itemId, itemId, row) + acceptJournalSubmissionFromProviderItem(state, mutation.itemId, itemId, row) upsertItem( state, itemId, @@ -124,7 +128,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo applySubmission(state, row) return } - applyDispatch(state, row) + applyJournalDispatchRow(state, row) } export function rememberAppliedSettlementId( @@ -265,76 +269,6 @@ function applySubmission( upsertItem(state, itemId, 0, journalRenderItem(itemId, 0, row.body, row)) } -function applyDispatch( - state: JournalReducerState, - row: Extract -): void { - const submission = state.submissions.get(row.clientMessageId) - if (!submission) { - return - } - // `rejected` is terminal; a late `unknown` must not reopen a settled answer. - if (submission.dispatchState === 'rejected' || submission.dispatchState === 'accepted') { - return - } - submission.fence = row.fence - submission.dispatchState = row.state - submission.providerItemId = row.providerItemId - submission.reason = row.reason - submission.resolvedAt = row.state === 'pending' ? null : row.ts - if (row.state === 'pending') { - submission.handedOverAt = row.ts - } - if (row.recovered) { - submission.recovered = row.recovered - } else { - delete submission.recovered - } - if (row.state !== 'accepted' || !row.providerItemId) { - return - } - state.aliases.set(row.providerItemId, agentJournalSubmissionKey(row.clientMessageId)) - state.receipts.set(row.clientMessageId, { - clientMessageId: row.clientMessageId, - providerItemId: row.providerItemId, - cursor: { epoch: row.epoch, sequence: row.seq }, - acceptedAt: row.ts - }) -} - -function acceptSubmissionFromProviderItem( - state: JournalReducerState, - providerItemId: string, - resolvedItemId: string, - row: Pick -): void { - if (providerItemId === resolvedItemId) { - return - } - const submission = [...state.submissions.values()].find( - (candidate) => agentJournalSubmissionKey(candidate.clientMessageId) === resolvedItemId - ) - if ( - !submission || - submission.dispatchState === 'accepted' || - submission.dispatchState === 'rejected' - ) { - return - } - submission.fence = row.fence - submission.dispatchState = 'accepted' - submission.providerItemId = providerItemId - submission.reason = null - submission.resolvedAt = row.ts - delete submission.recovered - state.receipts.set(submission.clientMessageId, { - clientMessageId: submission.clientMessageId, - providerItemId, - cursor: { epoch: row.epoch, sequence: row.seq }, - acceptedAt: row.ts - }) -} - /** Project the folded state into the client-facing snapshot. */ export function renderJournalState(state: JournalReducerState): AgentJournalSnapshot { // Sequence is the sole ordering key; map insertion order is not, because a diff --git a/src/main/native-chat/agent-session-journal/journal-submission-dispatch-state.ts b/src/main/native-chat/agent-session-journal/journal-submission-dispatch-state.ts new file mode 100644 index 000000000000..97b0255efa09 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-submission-dispatch-state.ts @@ -0,0 +1,75 @@ +// How a dispatch row, or the provider's echo of a send, moves a submission's delivery state. + +import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' +import type { JournalReducerState } from './journal-reducer' +import type { JournalRow } from './journal-row-schema' + +export function applyJournalDispatchRow( + state: JournalReducerState, + row: Extract +): void { + const submission = state.submissions.get(row.clientMessageId) + if (!submission) { + return + } + // `rejected` is terminal; a late `unknown` must not reopen a settled answer. + if (submission.dispatchState === 'rejected' || submission.dispatchState === 'accepted') { + return + } + submission.fence = row.fence + submission.dispatchState = row.state + submission.providerItemId = row.providerItemId + submission.reason = row.reason + submission.resolvedAt = row.state === 'pending' ? null : row.ts + if (row.state === 'pending') { + submission.handedOverAt = row.ts + } + if (row.recovered) { + submission.recovered = row.recovered + } else { + delete submission.recovered + } + if (row.state !== 'accepted' || !row.providerItemId) { + return + } + state.aliases.set(row.providerItemId, agentJournalSubmissionKey(row.clientMessageId)) + state.receipts.set(row.clientMessageId, { + clientMessageId: row.clientMessageId, + providerItemId: row.providerItemId, + cursor: { epoch: row.epoch, sequence: row.seq }, + acceptedAt: row.ts + }) +} + +export function acceptJournalSubmissionFromProviderItem( + state: JournalReducerState, + providerItemId: string, + resolvedItemId: string, + row: Pick +): void { + if (providerItemId === resolvedItemId) { + return + } + const submission = [...state.submissions.values()].find( + (candidate) => agentJournalSubmissionKey(candidate.clientMessageId) === resolvedItemId + ) + if ( + !submission || + submission.dispatchState === 'accepted' || + submission.dispatchState === 'rejected' + ) { + return + } + submission.fence = row.fence + submission.dispatchState = 'accepted' + submission.providerItemId = providerItemId + submission.reason = null + submission.resolvedAt = row.ts + delete submission.recovered + state.receipts.set(submission.clientMessageId, { + clientMessageId: submission.clientMessageId, + providerItemId, + cursor: { epoch: row.epoch, sequence: row.seq }, + acceptedAt: row.ts + }) +} From 27619ee7882ec6c3cda7347e8244263dc5d7a9ea Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:17:42 -0700 Subject: [PATCH 09/24] test(native-chat): a send or reader that opens a crashed chat settles a proven death interrupted Main's open-time settle test still asserted the old rule, where only a watched exit proved a death. --- ...agent-session-send-open-stale-turn.test.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts index ffd06ae4dace..8cdecdaa0308 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -81,7 +81,6 @@ async function relaunchAfterCrashMidTurn(probe: AgentSessionOwnerProbe) { return { host, acquire } } -/** Neither proof of the old owner's death is an observed exit, so the turn ends `unverifiable`. */ function turnStates(host: StructuredAgentSessionHost) { return host .journalSnapshot(SESSION) @@ -90,18 +89,20 @@ function turnStates(host: StructuredAgentSessionHost) { } // A host that cannot prove the old owner gone leaves its lease in recovery, still claimed, until -// the next acquire resolves it: the open is not that acquire, and the turn is no less gone. +// the next acquire resolves it: the open is not that acquire, and the turn is no less gone. Only +// the proof decides whether it reads interrupted. const PROBES = [ - ['the old owner is proven gone', { outcome: 'pid-absent' }], + ['the old owner is proven gone', { outcome: 'pid-absent' }, 'interrupted'], [ 'nothing proves the old owner gone', - { outcome: 'indeterminate', reason: 'This host cannot probe structured session owners.' } + { outcome: 'indeterminate', reason: 'This host cannot probe structured session owners.' }, + 'unverifiable' ] -] as const satisfies readonly (readonly [string, AgentSessionOwnerProbe])[] +] as const satisfies readonly (readonly [string, AgentSessionOwnerProbe, string])[] it.each(PROBES)( 'settles a turn a dead generation left running when a send opens the chat and its start fails, when %s', - async (_when, probe) => { + async (_when, probe, settled) => { const { host, acquire } = await relaunchAfterCrashMidTurn(probe) expect(host.hasSession(SESSION)).toBe(false) @@ -119,19 +120,19 @@ it.each(PROBES)( ) expect(acquire).toHaveBeenCalledOnce() - expect(turnStates(host)).toEqual(['unverifiable']) + expect(turnStates(host)).toEqual([settled]) await host.flushAllStreamedEvents() } ) it.each(PROBES)( 'settles the same turn when a reader opens the chat, when %s', - async (_when, probe) => { + async (_when, probe, settled) => { const { host } = await relaunchAfterCrashMidTurn(probe) await host.revealSession(SESSION) - expect(turnStates(host)).toEqual(['unverifiable']) + expect(turnStates(host)).toEqual([settled]) await host.flushAllStreamedEvents() } ) From 11b1a913c16eb559ff6ad6bd1b3bc66f388e7f33 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:21:00 -0700 Subject: [PATCH 10/24] docs(native-chat): say which proofs of death record a last proof of life --- src/shared/agent-session-record.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shared/agent-session-record.ts b/src/shared/agent-session-record.ts index bd0e166437a8..7487c5944c71 100644 --- a/src/shared/agent-session-record.ts +++ b/src/shared/agent-session-record.ts @@ -88,7 +88,8 @@ export type AgentSessionDeathEvidence = { detail: string observedAt: number /** The death interval's lower bound: the last time the runtime holding the owner's transport - * proved it alive. Absent when nothing did, and on evidence older builds wrote. */ + * proved it alive. Only a probe's proof records it: absent on a surface-release exit, a failed + * start, and evidence older builds wrote. */ lastProvenAliveAt?: number } From b22c1b87b72f941448be051645f727cdebc99718 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:36:37 -0700 Subject: [PATCH 11/24] fix(native-chat): a proof of life bounds only the owner that wrote the turn A start after a crash that spawned a child and then failed without proving it gone parks that child for recovery; when recovery finds it gone, the proof of death carries its proof of life, which is after the crash. The older turn then ended there and counted the downtime. The journal now derives the fence of its newest live writer, and the lease that holds the proof names the owner it released by the fence it moved to. The last renewal counts only when that move was one step past the writer of the turn. --- .../claude-context-usage-restart.test.ts | 4 +- .../agent-session-journal/journal-reducer.ts | 8 ++- .../agent-session-journal/journal-store.ts | 2 + ...ured-agent-session-attach-orchestration.ts | 4 +- ...uctured-agent-session-conversation-open.ts | 2 +- ...tured-agent-session-crash-turn-end.test.ts | 71 +++++++++++++++++++ ...gent-session-dead-generation-settlement.ts | 10 +-- ...agent-session-recovered-turn-clock.test.ts | 2 +- ...d-agent-session-stale-turn-verdict.test.ts | 69 +++++++++++++----- ...ctured-agent-session-stale-turn-verdict.ts | 25 +++++-- ...ured-agent-session-unexpected-exit.test.ts | 2 +- src/shared/agent-session-next-fence.ts | 4 +- 12 files changed, 165 insertions(+), 38 deletions(-) diff --git a/src/main/claude/claude-context-usage-restart.test.ts b/src/main/claude/claude-context-usage-restart.test.ts index cb0dff927c50..8d1c9001a92d 100644 --- a/src/main/claude/claude-context-usage-restart.test.ts +++ b/src/main/claude/claude-context-usage-restart.test.ts @@ -217,7 +217,7 @@ describe('context usage across a restart', () => { sessionId: 'orca-session', fence: 2, acquisitionGeneration: 'next', - deathEvidence: null + deathRecord: null }) const turns = journal.snapshot().items.map((item) => readAgentJournalTurn(item.body)?.state) expect(turns).toContain('unverifiable') @@ -239,7 +239,7 @@ describe('context usage across a restart', () => { sessionId: 'orca-session', fence: 1, acquisitionGeneration: 'next', - deathEvidence: null + deathRecord: null }) live.reattach() await live.settle() diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index f5c9f539e779..d97dbc919d29 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -41,6 +41,8 @@ export type JournalReducerState = { lastActivityAt: number /** `lastActivityAt` over rows written live: crash reconciliation is not the provider working. */ lastLiveActivityAt: number + /** Fence of the newest live writer: the generation a live bound describes. */ + lastLiveFence: number /** Lowest sequence still individually replayable; rows below it were compacted. */ oldestSequence: number highestFence: number @@ -62,6 +64,7 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou lastSequence: 0, lastActivityAt: 0, lastLiveActivityAt: 0, + lastLiveFence: 0, oldestSequence: 1, highestFence: 0, items: new Map(), @@ -80,7 +83,10 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } state.lastActivityAt = Math.max(state.lastActivityAt, row.ts) - state.lastLiveActivityAt = Math.max(state.lastLiveActivityAt, row.recovered ? 0 : row.ts) + if (!row.recovered) { + state.lastLiveActivityAt = Math.max(state.lastLiveActivityAt, row.ts) + state.lastLiveFence = Math.max(state.lastLiveFence, row.fence) + } if (row.kind === 'item') { if (journalItemRevisionIsStale(state, row.itemId, row.revision)) { return diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 1bb7753df582..bd9a0942b7d5 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -236,6 +236,8 @@ export class AgentSessionJournal { /** The same over rows written live, so crash reconciliation never reads as activity; 0 if none. */ lastLiveActivityAt = (): number => this.state.lastLiveActivityAt + lastLiveFence = (): number => this.state.lastLiveFence + submissions = (): AgentJournalSubmission[] => [...this.state.submissions.values()] pendingSubmissions = (): AgentJournalSubmission[] => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index 591c09d8e32b..90757a7d51a5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -129,7 +129,7 @@ async function runAttach( const attemptSink = context.runtimeState.mintEventSink(sessionId) // Read before the reserve clears it: how the previous generation ended decides how whatever it // left running is settled. - const priorDeathEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null + const priorDeathRecord = context.deps.store.getRecord(sessionId)?.lease ?? null const attempt: { candidate: AttachCandidate | null; committed: boolean } = { candidate: null, committed: false @@ -185,7 +185,7 @@ async function runAttach( sessionId, fence, acquisitionGeneration, - deathEvidence: priorDeathEvidence + deathRecord: priorDeathRecord }) } await bindAndDrain(eventSink, attached.journal, fence, (activity) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index 07d1fb6b5ffb..f9c1e51ed514 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -112,7 +112,7 @@ export async function openStructuredAgentSessionConversationJournal( sessionId, fence, acquisitionGeneration: null, - deathEvidence: record.lease.deathEvidence ?? null + deathRecord: record.lease }) } } catch (error) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index 519ce1165fa9..bee860b8135c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -19,9 +19,14 @@ import { AgentSessionRecordStore } from '../../runtime/agent-session-record-stor import { AGENT_SESSION_STORE_FILE_NAME } from '../../runtime/agent-session-record-store-file' import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { + AgentSessionAcquisitionExitUnprovenError, + type StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' import { StructuredAgentSessionHost } from './structured-agent-session-host' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { + hostTestAttachParams, HOST_TEST_LOCATION as LOCATION, HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' @@ -217,4 +222,70 @@ describe('a turn a crash cut short mid-tool', () => { }) expect(settledTurn()).toMatchObject({ state: 'interrupted', completedAt: LAST_RENEWED_AT }) }) + + it('never takes a proof of life from a newer owner a failed start left for recovery', async () => { + // A start after the relaunch spawned a child and failed without proving it gone, so it was + // parked; a later start found that child gone. Its proof of life is from after the crash. + let now = RELAUNCHED_AT + const claudeHandle = { kind: 'claude' as const, sessionId: PROVIDER_SESSION, leafUuid: null } + const acquire = vi.fn( + async ({ fence, spawnToken, onSpawned }) => { + const process = { hostId: 'local', pid: 7_000 + fence, processStartTimeMs: now, spawnToken } + await onSpawned?.(process) + if (fence === 15) { + throw new AgentSessionAcquisitionExitUnprovenError(new Error('hung')) + } + return { + process, + link: { + linkId: `claude-${fence}-link`, + handle: { provider: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null }, + origin: 'resumed', + mintedAtFence: fence, + observedAt: now + } + } + } + ) + openHost({ + adapter: { + acquire, + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + probeOwner: async () => ({ outcome: 'pid-absent' }), + now: () => now + }) + await host.reconcileRestartLeases() + const attach = (fence: number) => + host.attach( + { callerKey: 'client-1' }, + hostTestAttachParams(fence, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + providerHandle: claudeHandle + }) + ) + + await attach(14).catch(() => undefined) + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + runtimeFence: 15, + handoffStage: 'recovering', + lastRenewedAt: RELAUNCHED_AT + }) + now = RELAUNCHED_AT + 60_000 + // Recovery records that child's death, then the client retries at the fence it was told. + await expect(attach(15)).resolves.toMatchObject({ refusal: { currentFence: 16 } }) + expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ + lastProvenAliveAt: RELAUNCHED_AT + }) + await expect(attach(16)).resolves.toMatchObject({ ok: true }) + + // The owner that wrote the turn left no proof of life to use, so its last row bounds it. + expect(settledTurn()).toMatchObject({ state: 'interrupted', completedAt: TOOL_STARTED_AT }) + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index a17d22e41b65..47be184ccb1e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -12,10 +12,10 @@ import { } from '../agent-session-journal/journal-prompt-body-bounds' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' -import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { runningTurnLifecycleRevisions, turnVerdictFromDeathEvidence, + type StructuredAgentSessionDeathRecord, type StructuredAgentSessionTurnVerdict } from './structured-agent-session-stale-turn-verdict' @@ -230,11 +230,13 @@ export async function settleStaleStructuredAgentSessionState(input: { sessionId: string fence: number acquisitionGeneration: string | null - deathEvidence: AgentSessionDeathEvidence | null + /** The lease as it stood before any new owner reserved it. */ + deathRecord: StructuredAgentSessionDeathRecord | null }): Promise { const { journal } = input const items = journal.snapshot().items - const verdict = turnVerdictFromDeathEvidence(input.deathEvidence, journal) + const deathEvidence = input.deathRecord?.deathEvidence ?? null + const verdict = turnVerdictFromDeathEvidence(input.deathRecord, journal) const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` const mutations: JournalLifecycleMutationInput[] = [] @@ -255,7 +257,7 @@ export async function settleStaleStructuredAgentSessionState(input: { // Only a watched exit carries the provider's own reason; a probe's detail is Orca's. text: boundJournalStatusText( unexpectedProviderExitOutcome( - input.deathEvidence?.kind === 'exit-observed' ? input.deathEvidence.detail : undefined + deathEvidence?.kind === 'exit-observed' ? deathEvidence.detail : undefined ) ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts index 72a908a3361c..32632e45783b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts @@ -162,7 +162,7 @@ describe('a turn recovery settled after its host went away', () => { sessionId: SESSION, fence: 2, acquisitionGeneration: 'generation-2', - deathEvidence: null + deathRecord: null }) session.publish() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 59b13c8c5798..3ad4d4d932f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { codexSubagentGroupBody, @@ -82,13 +83,19 @@ function promptItem(state: 'pending' | 'resolved', sequence: number): AgentJourn } } +/** The lease a release of the fence-1 owner leaves behind: its fence moved one past. */ +function released(deathEvidence: AgentSessionDeathEvidence | null, runtimeFence = 2) { + return { deathEvidence, runtimeFence } +} + describe('turn verdict from death evidence', () => { - const lastLiveAt = (at: number) => ({ lastLiveActivityAt: () => at }) + /** A journal whose last live row, written by the fence-1 owner, is at `at`. */ + const lastLiveAt = (at: number) => ({ lastLiveActivityAt: () => at, lastLiveFence: () => 1 }) it('ends a watched exit at the exit', () => { expect( turnVerdictFromDeathEvidence( - { kind: 'exit-observed', detail: 'exit', observedAt: 500 }, + released({ kind: 'exit-observed', detail: 'exit', observedAt: 500 }), lastLiveAt(300) ) ).toEqual({ state: 'interrupted', completedAt: 500 }) @@ -99,13 +106,16 @@ describe('turn verdict from death evidence', () => { (kind) => { // Probed at 9000, long after the crash: the downtime is never counted as work. expect( - turnVerdictFromDeathEvidence({ kind, detail: 'gone', observedAt: 9_000 }, lastLiveAt(300)) + turnVerdictFromDeathEvidence( + released({ kind, detail: 'gone', observedAt: 9_000 }), + lastLiveAt(300) + ) ).toEqual({ state: 'interrupted', completedAt: 300 }) // A live row stamped after the probe cannot outlast it, and no live row leaves only the probe. for (const lastLive of [9_500, 0]) { expect( turnVerdictFromDeathEvidence( - { kind, detail: 'gone', observedAt: 9_000 }, + released({ kind, detail: 'gone', observedAt: 9_000 }), lastLiveAt(lastLive) ) ).toEqual({ state: 'interrupted', completedAt: 9_000 }) @@ -114,12 +124,8 @@ describe('turn verdict from death evidence', () => { ) it('ends a probe-proven death at the later of the last renewal and the last live row', () => { - const proof = (lastProvenAliveAt: number) => ({ - kind: 'pid-absent' as const, - detail: 'gone', - observedAt: 9_000, - lastProvenAliveAt - }) + const proof = (lastProvenAliveAt: number) => + released({ kind: 'pid-absent', detail: 'gone', observedAt: 9_000, lastProvenAliveAt }) // A silent tool run: the renewal saw the child working long after its last row. expect(turnVerdictFromDeathEvidence(proof(8_000), lastLiveAt(300))).toEqual({ state: 'interrupted', @@ -141,10 +147,32 @@ describe('turn verdict from death evidence', () => { }) }) + it('ignores a renewal recorded for a newer owner than the one that wrote the turn', () => { + // Fence 1 wrote the turn and was released to 2; a start reserved 3 and proved its child alive + // at 8000, long after the crash, and the release that recorded its death moved to 4. + const newer = released( + { kind: 'pid-absent', detail: 'gone', observedAt: 9_000, lastProvenAliveAt: 8_000 }, + 4 + ) + expect(turnVerdictFromDeathEvidence(newer, lastLiveAt(300))).toEqual({ + state: 'interrupted', + completedAt: 300 + }) + // A fence floor from a store recovered from its backup is still one move. + expect( + turnVerdictFromDeathEvidence({ ...newer, minimumNextFence: 4 }, lastLiveAt(300)) + ).toEqual({ state: 'interrupted', completedAt: 8_000 }) + }) + it('ends a watched exit at the exit even when a renewal is recorded', () => { expect( turnVerdictFromDeathEvidence( - { kind: 'exit-observed', detail: 'exit', observedAt: 500, lastProvenAliveAt: 400 }, + released({ + kind: 'exit-observed', + detail: 'exit', + observedAt: 500, + lastProvenAliveAt: 400 + }), lastLiveAt(300) ) ).toEqual({ state: 'interrupted', completedAt: 500 }) @@ -152,6 +180,9 @@ describe('turn verdict from death evidence', () => { it('leaves a release nothing proved unverifiable', () => { expect(turnVerdictFromDeathEvidence(null, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) + expect(turnVerdictFromDeathEvidence(released(null), lastLiveAt(300))).toEqual({ + state: 'unverifiable' + }) expect(turnVerdictFromDeathEvidence(undefined, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) @@ -272,7 +303,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: 'generation-2', - deathEvidence: null + deathRecord: null }) ).resolves.toBe(1) @@ -301,7 +332,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: 'generation-2', - deathEvidence: null + deathRecord: null }) ).resolves.toBe(1) @@ -364,7 +395,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 2, acquisitionGeneration: 'generation-2', - deathEvidence: null + deathRecord: null }) expect( @@ -432,11 +463,11 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 2, acquisitionGeneration: 'generation-2', - deathEvidence: { + deathRecord: released({ kind: 'pid-absent', detail: 'recorded pid absent on host', observedAt: 9_000 - } + }) }) const items = journal.snapshot().items @@ -491,7 +522,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 2, acquisitionGeneration: 'generation-2', - deathEvidence: { kind: 'pid-absent', detail: 'gone', observedAt: 8_500 } + deathRecord: released({ kind: 'pid-absent', detail: 'gone', observedAt: 8_500 }) }) expect( @@ -516,7 +547,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: null, - deathEvidence: null + deathRecord: null }) ).resolves.toBe(0) expect(idle.appendLifecycleBatch).not.toHaveBeenCalled() @@ -527,7 +558,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: null, - deathEvidence: null + deathRecord: null }) expect(running.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ settlementId: 'stale-session:session-1:14:seq-8' }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 4d0c502a70db..550c073fa7a0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -14,7 +14,8 @@ import { agentJournalTurnBody, readAgentJournalTurn } from '../../../shared/agent-session-turn-record' -import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import { nextAgentSessionFence } from '../../../shared/agent-session-next-fence' +import type { AgentSessionLease } from '../../../shared/agent-session-record' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' @@ -26,19 +27,31 @@ export const UNVERIFIABLE_TURN_VERDICT: StructuredAgentSessionTurnVerdict = { state: 'unverifiable' } +/** The lease as read with its death evidence: the fence its release moved to names the owner. */ +export type StructuredAgentSessionDeathRecord = Pick< + AgentSessionLease, + 'deathEvidence' | 'runtimeFence' | 'minimumNextFence' +> + export function turnVerdictFromDeathEvidence( - evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick + lease: StructuredAgentSessionDeathRecord | null | undefined, + journal: Pick ): StructuredAgentSessionTurnVerdict { - if (!evidence) { + const evidence = lease?.deathEvidence + if (!lease || !evidence) { return UNVERIFIABLE_TURN_VERDICT } if (evidence.kind === 'exit-observed') { return { state: 'interrupted', completedAt: evidence.observedAt } } // A probe finds a dead child long after it died. The later of the last renewal and the last row - // it wrote bounds its end, so the turn never counts the time Orca itself was down. - const lastSeen = Math.max(evidence.lastProvenAliveAt ?? 0, journal.lastLiveActivityAt()) + // it wrote bounds its end, so the turn never counts the time Orca itself was down. The renewal + // counts only when the release that recorded it moved past the generation that wrote the turn. + const sameOwner = + lease.runtimeFence === + nextAgentSessionFence({ ...lease, runtimeFence: journal.lastLiveFence() }) + const lastProvenAliveAt = sameOwner ? (evidence.lastProvenAliveAt ?? 0) : 0 + const lastSeen = Math.max(lastProvenAliveAt, journal.lastLiveActivityAt()) return { state: 'interrupted', completedAt: lastSeen > 0 ? Math.min(lastSeen, evidence.observedAt) : evidence.observedAt diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 6419e1cfb1ad..d94a88ce0d26 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -179,7 +179,7 @@ describe('provider-exit recovery tickets', () => { sessionId: SESSION, fence: 8, acquisitionGeneration: 'generation-2', - deathEvidence: record.lease.deathEvidence + deathRecord: record.lease }) expect(appendLifecycleBatch.mock.calls.at(-1)?.[0].mutations).toContainEqual( expect.objectContaining({ diff --git a/src/shared/agent-session-next-fence.ts b/src/shared/agent-session-next-fence.ts index 2c05aa5004db..d5b2f2a08b3b 100644 --- a/src/shared/agent-session-next-fence.ts +++ b/src/shared/agent-session-next-fence.ts @@ -12,6 +12,8 @@ import type { AgentSessionLease } from './agent-session-record' -export function nextAgentSessionFence(lease: AgentSessionLease): number { +export function nextAgentSessionFence( + lease: Pick +): number { return Math.max(lease.runtimeFence + 1, lease.minimumNextFence ?? 0) } From 514c9d10bc3152501ddfd9c87318f0d873b3744a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:39:08 -0700 Subject: [PATCH 12/24] test(native-chat): a crash with a send in doubt still ends at the last proof of life The reopen settles that send at the new fence, so the owner check must read the fence of live rows only. --- ...structured-agent-session-crash-turn-end.test.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index bee860b8135c..2eaa4cc0fc7c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -105,7 +105,8 @@ async function seedCrashedStore(): Promise { store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) } -/** A running turn whose only row after its start is a Bash call that never reported back. */ +/** A running turn whose only row after its start is a Bash call that never reported back, for a + * send the provider never acknowledged: the reopen settles it at the fence after the crash. */ async function seedClaudeToolTurn(): Promise { let now = TOOL_STARTED_AT - 2_000 const journal = await openAgentSessionJournal({ @@ -122,11 +123,12 @@ async function seedClaudeToolTurn(): Promise { }), now: () => now }) - await journal.appendItem( - { provider: 'claude', sessionId: PROVIDER_SESSION, uuid: 'uuid-user' }, - { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'run the loop' }] }, - { fence: 13 } - ) + await journal.appendSubmission({ + clientMessageId: 'send-1', + payloadFingerprint: '0'.repeat(64), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'run the loop' }] }, + fence: 13 + }) await journal.appendItem( { provider: 'claude', sessionId: PROVIDER_SESSION, uuid: 'uuid-turn' }, { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: now }, From 63f6637bc3e753ccec90bab0e8f63981b753c69e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:58:00 -0700 Subject: [PATCH 13/24] fix(native-chat): a proof of death judges only the turn its own owner wrote The settle read the record's latest proof of death for whatever turn a gone generation left running. After a crash, a start that reserved a new fence cleared the relaunch's proof, and if it then failed, its own child's death (a watched exit at the failure, or a probe finding the child it left for recovery gone) ended the older turn an hour after the crash. Every proof of death now records ownerFence, the fence of the owner or reservation it is about; a fence names exactly one owner. The journal derives the fence each item was created at, and a running turn is interrupted only by a proof naming its own owner; otherwise it is unverifiable. Evidence older builds wrote keeps their rule. This replaces the derived one-step fence check. --- .../claude-context-usage-restart.test.ts | 4 +- .../agent-session-journal/journal-reducer.ts | 29 ++-- .../agent-session-journal/journal-store.ts | 3 +- ...ured-agent-session-attach-orchestration.ts | 4 +- ...uctured-agent-session-conversation-open.ts | 2 +- ...tured-agent-session-crash-turn-end.test.ts | 120 ++++++++++------ ...gent-session-dead-generation-settlement.ts | 18 +-- ...agent-session-recovered-turn-clock.test.ts | 2 +- ...d-agent-session-stale-turn-verdict.test.ts | 134 +++++++++--------- ...ctured-agent-session-stale-turn-verdict.ts | 45 +++--- ...ured-agent-session-unexpected-exit.test.ts | 3 +- ...-session-acquisition-failure-settlement.ts | 33 +++-- ...session-death-evidence-persistence.test.ts | 13 +- ...gent-session-surface-release-transition.ts | 3 +- .../agent-session-lease-adjudication.test.ts | 10 +- .../agent-session-lease-adjudication.ts | 13 +- src/shared/agent-session-next-fence.ts | 4 +- src/shared/agent-session-record.ts | 6 +- ...sion-death-evidence-downgrade.unit.test.ts | 61 ++++++++ 19 files changed, 318 insertions(+), 189 deletions(-) create mode 100644 tests/e2e/cross-version-wire/agent-session-death-evidence-downgrade.unit.test.ts diff --git a/src/main/claude/claude-context-usage-restart.test.ts b/src/main/claude/claude-context-usage-restart.test.ts index 8d1c9001a92d..cb0dff927c50 100644 --- a/src/main/claude/claude-context-usage-restart.test.ts +++ b/src/main/claude/claude-context-usage-restart.test.ts @@ -217,7 +217,7 @@ describe('context usage across a restart', () => { sessionId: 'orca-session', fence: 2, acquisitionGeneration: 'next', - deathRecord: null + deathEvidence: null }) const turns = journal.snapshot().items.map((item) => readAgentJournalTurn(item.body)?.state) expect(turns).toContain('unverifiable') @@ -239,7 +239,7 @@ describe('context usage across a restart', () => { sessionId: 'orca-session', fence: 1, acquisitionGeneration: 'next', - deathRecord: null + deathEvidence: null }) live.reattach() await live.settle() diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index d97dbc919d29..41fdf3654cc3 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -41,12 +41,12 @@ export type JournalReducerState = { lastActivityAt: number /** `lastActivityAt` over rows written live: crash reconciliation is not the provider working. */ lastLiveActivityAt: number - /** Fence of the newest live writer: the generation a live bound describes. */ - lastLiveFence: number /** Lowest sequence still individually replayable; rows below it were compacted. */ oldestSequence: number highestFence: number items: Map + /** Fence of the writer that created each item: the generation a running turn belongs to. */ + itemFences: Map /** Revision of a removed item, so a late lower revision cannot resurrect it. */ tombstones: Map submissions: Map @@ -64,10 +64,10 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou lastSequence: 0, lastActivityAt: 0, lastLiveActivityAt: 0, - lastLiveFence: 0, oldestSequence: 1, highestFence: 0, items: new Map(), + itemFences: new Map(), tombstones: new Map(), submissions: new Map(), receipts: new Map(), @@ -83,17 +83,20 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } state.lastActivityAt = Math.max(state.lastActivityAt, row.ts) - if (!row.recovered) { - state.lastLiveActivityAt = Math.max(state.lastLiveActivityAt, row.ts) - state.lastLiveFence = Math.max(state.lastLiveFence, row.fence) - } + state.lastLiveActivityAt = Math.max(state.lastLiveActivityAt, row.recovered ? 0 : row.ts) if (row.kind === 'item') { if (journalItemRevisionIsStale(state, row.itemId, row.revision)) { return } const itemId = resolveJournalItemId(state, row.itemId, row.body) acceptJournalSubmissionFromProviderItem(state, row.itemId, itemId, row) - upsertItem(state, itemId, row.revision, journalRenderItem(itemId, row.revision, row.body, row)) + upsertItem( + state, + itemId, + row.revision, + journalRenderItem(itemId, row.revision, row.body, row), + row.fence + ) return } if (row.kind === 'tombstone') { @@ -121,7 +124,8 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo mutation.body, row, journalBatchMutationProducer(row, mutation) - ) + ), + row.fence ) } else { removeItem(state, resolveItemId(state, mutation.itemId), mutation.revision) @@ -206,7 +210,8 @@ function upsertItem( state: JournalReducerState, itemId: string, revision: number, - next: AgentJournalRenderItem + next: AgentJournalRenderItem, + fence: number ): void { const tombstoned = state.tombstones.get(itemId) if (tombstoned !== undefined && revision <= tombstoned) { @@ -218,6 +223,7 @@ function upsertItem( } if (!existing) { state.items.set(itemId, next) + state.itemFences.set(itemId, fence) state.tombstones.delete(itemId) return } @@ -254,6 +260,7 @@ function removeItem(state: JournalReducerState, itemId: string, revision: number } state.tombstones.set(itemId, revision) state.items.delete(itemId) + state.itemFences.delete(itemId) } function applySubmission( @@ -272,7 +279,7 @@ function applySubmission( ...(row.handoverRecorded ? { handoverRecorded: true, acceptedSequence: row.seq } : {}) }) const itemId = agentJournalSubmissionKey(row.clientMessageId) - upsertItem(state, itemId, 0, journalRenderItem(itemId, 0, row.body, row)) + upsertItem(state, itemId, 0, journalRenderItem(itemId, 0, row.body, row), row.fence) } /** Project the folded state into the client-facing snapshot. */ diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index bd9a0942b7d5..8ae12592f23f 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -236,7 +236,8 @@ export class AgentSessionJournal { /** The same over rows written live, so crash reconciliation never reads as activity; 0 if none. */ lastLiveActivityAt = (): number => this.state.lastLiveActivityAt - lastLiveFence = (): number => this.state.lastLiveFence + /** Fence of the writer that created the item, while it is in the timeline. */ + itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) submissions = (): AgentJournalSubmission[] => [...this.state.submissions.values()] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index 90757a7d51a5..591c09d8e32b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -129,7 +129,7 @@ async function runAttach( const attemptSink = context.runtimeState.mintEventSink(sessionId) // Read before the reserve clears it: how the previous generation ended decides how whatever it // left running is settled. - const priorDeathRecord = context.deps.store.getRecord(sessionId)?.lease ?? null + const priorDeathEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null const attempt: { candidate: AttachCandidate | null; committed: boolean } = { candidate: null, committed: false @@ -185,7 +185,7 @@ async function runAttach( sessionId, fence, acquisitionGeneration, - deathRecord: priorDeathRecord + deathEvidence: priorDeathEvidence }) } await bindAndDrain(eventSink, attached.journal, fence, (activity) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index f9c1e51ed514..07d1fb6b5ffb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -112,7 +112,7 @@ export async function openStructuredAgentSessionConversationJournal( sessionId, fence, acquisitionGeneration: null, - deathRecord: record.lease + deathEvidence: record.lease.deathEvidence ?? null }) } } catch (error) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index 2eaa4cc0fc7c..09ec52660428 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -224,54 +224,76 @@ describe('a turn a crash cut short mid-tool', () => { }) expect(settledTurn()).toMatchObject({ state: 'interrupted', completedAt: LAST_RENEWED_AT }) }) +}) - it('never takes a proof of life from a newer owner a failed start left for recovery', async () => { - // A start after the relaunch spawned a child and failed without proving it gone, so it was - // parked; a later start found that child gone. Its proof of life is from after the crash. - let now = RELAUNCHED_AT - const claudeHandle = { kind: 'claude' as const, sessionId: PROVIDER_SESSION, leafUuid: null } - const acquire = vi.fn( - async ({ fence, spawnToken, onSpawned }) => { - const process = { hostId: 'local', pid: 7_000 + fence, processStartTimeMs: now, spawnToken } - await onSpawned?.(process) - if (fence === 15) { - throw new AgentSessionAcquisitionExitUnprovenError(new Error('hung')) - } - return { - process, - link: { - linkId: `claude-${fence}-link`, - handle: { provider: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null }, - origin: 'resumed', - mintedAtFence: fence, - observedAt: now - } +/** A relaunched host whose first start after the crash, at fence 15, fails with `failure`; every + * later start succeeds. The crashed owner, and any child left behind, probe gone. */ +async function hostWithFailingFirstStart(failure: Error) { + let now = RELAUNCHED_AT + const acquire = vi.fn( + async ({ fence, spawnToken, onSpawned }) => { + const process = { hostId: 'local', pid: 7_000 + fence, processStartTimeMs: now, spawnToken } + await onSpawned?.(process) + if (fence === 15) { + throw failure + } + return { + process, + link: { + linkId: `claude-${fence}-link`, + handle: { provider: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null }, + origin: 'resumed', + mintedAtFence: fence, + observedAt: now } } - ) - openHost({ - adapter: { - acquire, - dispatch: vi.fn(), - cancelTurn: vi.fn(), - answerPrompt: vi.fn(), - setOption: vi.fn(), - supportsCreate: () => true - }, - probeOwner: async () => ({ outcome: 'pid-absent' }), - now: () => now - }) - await host.reconcileRestartLeases() - const attach = (fence: number) => + } + ) + openHost({ + adapter: { + acquire, + releaseAcquisition: async () => true, + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + probeOwner: async () => ({ outcome: 'pid-absent' }), + now: () => now + }) + await host.reconcileRestartLeases() + return { + attach: (fence: number) => host.attach( { callerKey: 'client-1' }, hostTestAttachParams(fence, { provider: 'claude', agent: 'claude', accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, - providerHandle: claudeHandle + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null } }) - ) + ), + advance: (ms: number) => { + now += ms + } + } +} + +/** The turn as the crashed owner left it, with no end: nothing proves when that owner stopped. */ +const UNVERIFIABLE_TURN = { + turnId: 'turn-1', + state: 'unverifiable', + startedAt: TOOL_STARTED_AT - 2_000 +} + +// The relaunch proved the fence-13 owner gone, but a start reserving fence 15 clears that proof +// before the turn is settled; what the record holds afterwards is about the start's own child. +describe('a turn a newer start could not settle before it failed', () => { + it('is not judged by the death of the child it left for recovery', async () => { + const { attach, advance } = await hostWithFailingFirstStart( + new AgentSessionAcquisitionExitUnprovenError(new Error('hung')) + ) await attach(14).catch(() => undefined) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -279,15 +301,31 @@ describe('a turn a crash cut short mid-tool', () => { handoffStage: 'recovering', lastRenewedAt: RELAUNCHED_AT }) - now = RELAUNCHED_AT + 60_000 + advance(60_000) // Recovery records that child's death, then the client retries at the fence it was told. await expect(attach(15)).resolves.toMatchObject({ refusal: { currentFence: 16 } }) expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ + ownerFence: 15, lastProvenAliveAt: RELAUNCHED_AT }) await expect(attach(16)).resolves.toMatchObject({ ok: true }) - // The owner that wrote the turn left no proof of life to use, so its last row bounds it. - expect(settledTurn()).toMatchObject({ state: 'interrupted', completedAt: TOOL_STARTED_AT }) + expect(settledTurn()).toEqual(UNVERIFIABLE_TURN) + }) + + it('is not judged by the watched exit of a start that failed', async () => { + const { attach, advance } = await hostWithFailingFirstStart(new Error('claude exited (code 1)')) + + await expect(attach(14)).resolves.toMatchObject({ ok: false }) + expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ + kind: 'exit-observed', + ownerFence: 15, + observedAt: RELAUNCHED_AT + }) + advance(60_000) + await expect(attach(16)).resolves.toMatchObject({ ok: true }) + + // Main ended it at the failed start, an hour after the crash, with the start's reason. + expect(settledTurn()).toEqual(UNVERIFIABLE_TURN) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 47be184ccb1e..0db75c264429 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -12,10 +12,10 @@ import { } from '../agent-session-journal/journal-prompt-body-bounds' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { runningTurnLifecycleRevisions, turnVerdictFromDeathEvidence, - type StructuredAgentSessionDeathRecord, type StructuredAgentSessionTurnVerdict } from './structured-agent-session-stale-turn-verdict' @@ -230,13 +230,13 @@ export async function settleStaleStructuredAgentSessionState(input: { sessionId: string fence: number acquisitionGeneration: string | null - /** The lease as it stood before any new owner reserved it. */ - deathRecord: StructuredAgentSessionDeathRecord | null + deathEvidence: AgentSessionDeathEvidence | null }): Promise { const { journal } = input const items = journal.snapshot().items - const deathEvidence = input.deathRecord?.deathEvidence ?? null - const verdict = turnVerdictFromDeathEvidence(input.deathRecord, journal) + // Each turn is judged by the evidence only if it names that turn's owner. + const verdictFor = (item: AgentJournalRenderItem) => + turnVerdictFromDeathEvidence(input.deathEvidence, journal, journal.itemFence(item.itemId)) const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` const mutations: JournalLifecycleMutationInput[] = [] @@ -247,8 +247,10 @@ export async function settleStaleStructuredAgentSessionState(input: { mutations.push({ kind: 'item', identity, body }) } } - mutations.push(...runningTurnLifecycleRevisions(items, verdict)) - if (verdict.state === 'interrupted' && items.some(isInProgressItem)) { + mutations.push( + ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))) + ) + if (items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted')) { mutations.unshift({ kind: 'item', identity: { provider: 'orca', clientMessageId: settlementId }, @@ -257,7 +259,7 @@ export async function settleStaleStructuredAgentSessionState(input: { // Only a watched exit carries the provider's own reason; a probe's detail is Orca's. text: boundJournalStatusText( unexpectedProviderExitOutcome( - deathEvidence?.kind === 'exit-observed' ? deathEvidence.detail : undefined + input.deathEvidence?.kind === 'exit-observed' ? input.deathEvidence.detail : undefined ) ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts index 32632e45783b..72a908a3361c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts @@ -162,7 +162,7 @@ describe('a turn recovery settled after its host went away', () => { sessionId: SESSION, fence: 2, acquisitionGeneration: 'generation-2', - deathRecord: null + deathEvidence: null }) session.publish() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 3ad4d4d932f9..05c34904a694 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -83,110 +83,110 @@ function promptItem(state: 'pending' | 'resolved', sequence: number): AgentJourn } } -/** The lease a release of the fence-1 owner leaves behind: its fence moved one past. */ -function released(deathEvidence: AgentSessionDeathEvidence | null, runtimeFence = 2) { - return { deathEvidence, runtimeFence } -} - describe('turn verdict from death evidence', () => { - /** A journal whose last live row, written by the fence-1 owner, is at `at`. */ - const lastLiveAt = (at: number) => ({ lastLiveActivityAt: () => at, lastLiveFence: () => 1 }) + const lastLiveAt = (at: number) => ({ lastLiveActivityAt: () => at }) + /** Judges a turn the fence-1 owner wrote, by evidence naming that owner. */ + const verdictForTurn = ( + evidence: AgentSessionDeathEvidence | null | undefined, + journal: ReturnType + ) => turnVerdictFromDeathEvidence(evidence && { ownerFence: 1, ...evidence }, journal, 1) it('ends a watched exit at the exit', () => { expect( - turnVerdictFromDeathEvidence( - released({ kind: 'exit-observed', detail: 'exit', observedAt: 500 }), - lastLiveAt(300) - ) + verdictForTurn({ kind: 'exit-observed', detail: 'exit', observedAt: 500 }, lastLiveAt(300)) ).toEqual({ state: 'interrupted', completedAt: 500 }) }) it.each(['pid-absent', 'identity-mismatch'] as const)( - 'ends a %s proof an older build recorded at the last row the journal saw live, not at the probe', + 'ends a %s proof with no proof of life at the last row the journal saw live, not at the probe', (kind) => { // Probed at 9000, long after the crash: the downtime is never counted as work. - expect( - turnVerdictFromDeathEvidence( - released({ kind, detail: 'gone', observedAt: 9_000 }), - lastLiveAt(300) - ) - ).toEqual({ state: 'interrupted', completedAt: 300 }) + expect(verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, lastLiveAt(300))).toEqual({ + state: 'interrupted', + completedAt: 300 + }) // A live row stamped after the probe cannot outlast it, and no live row leaves only the probe. for (const lastLive of [9_500, 0]) { expect( - turnVerdictFromDeathEvidence( - released({ kind, detail: 'gone', observedAt: 9_000 }), - lastLiveAt(lastLive) - ) + verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, lastLiveAt(lastLive)) ).toEqual({ state: 'interrupted', completedAt: 9_000 }) } } ) it('ends a probe-proven death at the later of the last renewal and the last live row', () => { - const proof = (lastProvenAliveAt: number) => - released({ kind: 'pid-absent', detail: 'gone', observedAt: 9_000, lastProvenAliveAt }) + const proof = (lastProvenAliveAt: number) => ({ + kind: 'pid-absent' as const, + detail: 'gone', + observedAt: 9_000, + lastProvenAliveAt + }) // A silent tool run: the renewal saw the child working long after its last row. - expect(turnVerdictFromDeathEvidence(proof(8_000), lastLiveAt(300))).toEqual({ + expect(verdictForTurn(proof(8_000), lastLiveAt(300))).toEqual({ state: 'interrupted', completedAt: 8_000 }) - expect(turnVerdictFromDeathEvidence(proof(8_000), lastLiveAt(0))).toEqual({ + expect(verdictForTurn(proof(8_000), lastLiveAt(0))).toEqual({ state: 'interrupted', completedAt: 8_000 }) // A chatty provider: its last row is the tighter bound. - expect(turnVerdictFromDeathEvidence(proof(200), lastLiveAt(300))).toEqual({ + expect(verdictForTurn(proof(200), lastLiveAt(300))).toEqual({ state: 'interrupted', completedAt: 300 }) // Neither bound outlasts the probe. - expect(turnVerdictFromDeathEvidence(proof(9_500), lastLiveAt(300))).toEqual({ + expect(verdictForTurn(proof(9_500), lastLiveAt(300))).toEqual({ state: 'interrupted', completedAt: 9_000 }) }) - it('ignores a renewal recorded for a newer owner than the one that wrote the turn', () => { - // Fence 1 wrote the turn and was released to 2; a start reserved 3 and proved its child alive - // at 8000, long after the crash, and the release that recorded its death moved to 4. - const newer = released( - { kind: 'pid-absent', detail: 'gone', observedAt: 9_000, lastProvenAliveAt: 8_000 }, - 4 - ) - expect(turnVerdictFromDeathEvidence(newer, lastLiveAt(300))).toEqual({ - state: 'interrupted', - completedAt: 300 - }) - // A fence floor from a store recovered from its backup is still one move. - expect( - turnVerdictFromDeathEvidence({ ...newer, minimumNextFence: 4 }, lastLiveAt(300)) - ).toEqual({ state: 'interrupted', completedAt: 8_000 }) - }) - it('ends a watched exit at the exit even when a renewal is recorded', () => { expect( - turnVerdictFromDeathEvidence( - released({ - kind: 'exit-observed', - detail: 'exit', - observedAt: 500, - lastProvenAliveAt: 400 - }), + verdictForTurn( + { kind: 'exit-observed', detail: 'exit', observedAt: 500, lastProvenAliveAt: 400 }, lastLiveAt(300) ) ).toEqual({ state: 'interrupted', completedAt: 500 }) }) it('leaves a release nothing proved unverifiable', () => { - expect(turnVerdictFromDeathEvidence(null, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) - expect(turnVerdictFromDeathEvidence(released(null), lastLiveAt(300))).toEqual({ - state: 'unverifiable' - }) - expect(turnVerdictFromDeathEvidence(undefined, lastLiveAt(300))).toEqual({ + expect(verdictForTurn(null, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) + expect(verdictForTurn(undefined, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) }) + it('keeps the rule an older build applied to evidence that names no owner', () => { + // Only a watched exit was proof then; a probe's proof stays unverifiable. + const legacy = { detail: 'gone', observedAt: 9_000, lastProvenAliveAt: 8_000 } + expect( + turnVerdictFromDeathEvidence({ ...legacy, kind: 'exit-observed' }, lastLiveAt(300), 1) + ).toEqual({ state: 'interrupted', completedAt: 9_000 }) + expect( + turnVerdictFromDeathEvidence({ ...legacy, kind: 'pid-absent' }, lastLiveAt(300), 1) + ).toEqual({ state: 'unverifiable' }) + }) + + it('gives a turn no end from evidence about any other owner', () => { + const proof = { + kind: 'pid-absent' as const, + detail: 'gone', + observedAt: 9_000, + lastProvenAliveAt: 8_000 + } + // A newer start's death, and a turn whose writer the timeline no longer knows: neither proves + // the turn's own owner gone. + for (const [evidence, turnFence] of [ + [{ ...proof, ownerFence: 3 }, 1], + [{ ...proof, kind: 'exit-observed' as const, ownerFence: 3 }, 1], + [{ ...proof, ownerFence: 1 }, undefined] + ] as const) { + expect(turnVerdictFromDeathEvidence(evidence, lastLiveAt(300), turnFence)).toEqual({ + state: 'unverifiable' + }) + } + }) }) describe('running turn lifecycle revisions', () => { @@ -285,6 +285,7 @@ describe('stale session state on a cold acquire', () => { const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 9 })) const journal = { snapshot: () => ({ items }), + itemFence: () => 1, cursor: () => ({ epoch: 'epoch-1', sequence: 8 }), appendLifecycleBatch } as unknown as AgentSessionJournal @@ -303,7 +304,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: 'generation-2', - deathRecord: null + deathEvidence: null }) ).resolves.toBe(1) @@ -332,7 +333,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: 'generation-2', - deathRecord: null + deathEvidence: null }) ).resolves.toBe(1) @@ -395,7 +396,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 2, acquisitionGeneration: 'generation-2', - deathRecord: null + deathEvidence: null }) expect( @@ -463,11 +464,12 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 2, acquisitionGeneration: 'generation-2', - deathRecord: released({ + deathEvidence: { kind: 'pid-absent', detail: 'recorded pid absent on host', - observedAt: 9_000 - }) + observedAt: 9_000, + ownerFence: 1 + } }) const items = journal.snapshot().items @@ -522,7 +524,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 2, acquisitionGeneration: 'generation-2', - deathRecord: released({ kind: 'pid-absent', detail: 'gone', observedAt: 8_500 }) + deathEvidence: { kind: 'pid-absent', detail: 'gone', observedAt: 8_500, ownerFence: 1 } }) expect( @@ -547,7 +549,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: null, - deathRecord: null + deathEvidence: null }) ).resolves.toBe(0) expect(idle.appendLifecycleBatch).not.toHaveBeenCalled() @@ -558,7 +560,7 @@ describe('stale session state on a cold acquire', () => { sessionId: 'session-1', fence: 14, acquisitionGeneration: null, - deathRecord: null + deathEvidence: null }) expect(running.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ settlementId: 'stale-session:session-1:14:seq-8' }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 550c073fa7a0..851b09d30209 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -1,9 +1,10 @@ // What the host may durably say about a turn whose provider child is gone. // -// `interrupted` requires proof the child is gone, which is exactly when a lease carries death -// evidence: a watched exit, or a local probe that found the recorded pid gone or reused. A release -// nothing proved — lost contact, an unverifiable identity, a stop that outlived the ladder — carries -// none, and its turn is `unverifiable` with no end at all. +// `interrupted` requires proof that the child which wrote the turn is gone: death evidence naming +// that turn's owner by fence — a watched exit, or a local probe that found the recorded pid gone or +// reused. A release nothing proved — lost contact, an unverifiable identity, a stop that outlived the +// ladder — carries none, and neither does a later owner's death; the turn is then `unverifiable` +// with no end at all. import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { @@ -14,8 +15,7 @@ import { agentJournalTurnBody, readAgentJournalTurn } from '../../../shared/agent-session-turn-record' -import { nextAgentSessionFence } from '../../../shared/agent-session-next-fence' -import type { AgentSessionLease } from '../../../shared/agent-session-record' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' @@ -27,31 +27,30 @@ export const UNVERIFIABLE_TURN_VERDICT: StructuredAgentSessionTurnVerdict = { state: 'unverifiable' } -/** The lease as read with its death evidence: the fence its release moved to names the owner. */ -export type StructuredAgentSessionDeathRecord = Pick< - AgentSessionLease, - 'deathEvidence' | 'runtimeFence' | 'minimumNextFence' -> - export function turnVerdictFromDeathEvidence( - lease: StructuredAgentSessionDeathRecord | null | undefined, - journal: Pick + evidence: AgentSessionDeathEvidence | null | undefined, + journal: Pick, + /** Fence of the owner that wrote the turn. */ + turnFence: number | undefined ): StructuredAgentSessionTurnVerdict { - const evidence = lease?.deathEvidence - if (!lease || !evidence) { + if (!evidence) { + return UNVERIFIABLE_TURN_VERDICT + } + if (evidence.ownerFence === undefined) { + // Evidence an older build wrote names no owner; it keeps the rule that build applied. + return evidence.kind === 'exit-observed' + ? { state: 'interrupted', completedAt: evidence.observedAt } + : UNVERIFIABLE_TURN_VERDICT + } + if (evidence.ownerFence !== turnFence) { return UNVERIFIABLE_TURN_VERDICT } if (evidence.kind === 'exit-observed') { return { state: 'interrupted', completedAt: evidence.observedAt } } // A probe finds a dead child long after it died. The later of the last renewal and the last row - // it wrote bounds its end, so the turn never counts the time Orca itself was down. The renewal - // counts only when the release that recorded it moved past the generation that wrote the turn. - const sameOwner = - lease.runtimeFence === - nextAgentSessionFence({ ...lease, runtimeFence: journal.lastLiveFence() }) - const lastProvenAliveAt = sameOwner ? (evidence.lastProvenAliveAt ?? 0) : 0 - const lastSeen = Math.max(lastProvenAliveAt, journal.lastLiveActivityAt()) + // it wrote bounds its end, so the turn never counts the time Orca itself was down. + const lastSeen = Math.max(evidence.lastProvenAliveAt ?? 0, journal.lastLiveActivityAt()) return { state: 'interrupted', completedAt: lastSeen > 0 ? Math.min(lastSeen, evidence.observedAt) : evidence.observedAt diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index d94a88ce0d26..07ae3150c22d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -136,6 +136,7 @@ describe('provider-exit recovery tickets', () => { snapshot: () => ({ items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })] }), + itemFence: () => 7, appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []) } @@ -179,7 +180,7 @@ describe('provider-exit recovery tickets', () => { sessionId: SESSION, fence: 8, acquisitionGeneration: 'generation-2', - deathRecord: record.lease + deathEvidence: record.lease.deathEvidence }) expect(appendLifecycleBatch.mock.calls.at(-1)?.[0].mutations).toContainEqual( expect.objectContaining({ diff --git a/src/main/runtime/agent-session-acquisition-failure-settlement.ts b/src/main/runtime/agent-session-acquisition-failure-settlement.ts index 2b28adc8b825..f6efaa890c37 100644 --- a/src/main/runtime/agent-session-acquisition-failure-settlement.ts +++ b/src/main/runtime/agent-session-acquisition-failure-settlement.ts @@ -103,18 +103,15 @@ export function settleFailedAgentSessionPostAcquisitionAttachment( claimStatus: 'released', lastRenewedAt: args.now, handoffOperationId: null, - deathEvidence: - args.exitProof === 'root-exit-observed' - ? { - kind: 'exit-observed', - detail: 'the provider process exited; its descendants were not proven gone', - observedAt: args.now - } - : { - kind: 'exit-observed', - detail: 'post-acquisition cleanup proved no provider child remains', - observedAt: args.now - } + deathEvidence: { + kind: 'exit-observed', + detail: + args.exitProof === 'root-exit-observed' + ? 'the provider process exited; its descendants were not proven gone' + : 'post-acquisition cleanup proved no provider child remains', + observedAt: args.now, + ownerFence: record.lease.runtimeFence + } }) state.records.set(args.sessionId, next) state.operations = settleAgentSessionOperation(state.operations, args) @@ -156,7 +153,7 @@ function settleFailedLease( claimStatus: 'released', lastRenewedAt: args.now, handoffOperationId: null, - deathEvidence: acquisitionDeathEvidence(args.exitProof, args.now) + deathEvidence: acquisitionDeathEvidence(args.exitProof, args.now, record.lease.runtimeFence) }) } @@ -164,25 +161,27 @@ function settleFailedLease( * when nothing was. */ function acquisitionDeathEvidence( exitProof: AgentSessionAcquisitionExitProof, - observedAt: number + observedAt: number, + ownerFence: number ): AgentSessionDeathEvidence | null { if (exitProof === 'unproven') { return null } + const proof = { observedAt, ownerFence } if (exitProof === 'processless') { - return { kind: 'pid-absent', detail: 'reservation failed before spawn', observedAt } + return { kind: 'pid-absent', detail: 'reservation failed before spawn', ...proof } } if (exitProof === 'root-exit-observed') { return { kind: 'exit-observed', detail: 'the provider process exited; its descendants were not proven gone', - observedAt + ...proof } } // Cleanup proved no child of this attempt remains; it may never have spawned. return { kind: 'exit-observed', detail: 'acquisition cleanup proved no provider child remains', - observedAt + ...proof } } diff --git a/src/main/runtime/agent-session-death-evidence-persistence.test.ts b/src/main/runtime/agent-session-death-evidence-persistence.test.ts index f00efa484a8b..1f17a1f40bbb 100644 --- a/src/main/runtime/agent-session-death-evidence-persistence.test.ts +++ b/src/main/runtime/agent-session-death-evidence-persistence.test.ts @@ -64,6 +64,7 @@ describe('death evidence on disk', () => { kind: 'pid-absent', detail: 'recorded pid absent on host', observedAt: 90_000, + ownerFence: 7, lastProvenAliveAt: LAST_RENEWED_AT } expect(crashed.getRecord(SESSION)?.lease.deathEvidence).toEqual(evidence) @@ -78,13 +79,11 @@ describe('death evidence on disk', () => { expect(store.getRecord(SESSION)?.lease.deathEvidence).toEqual(olderBuild) }) - it('quarantines evidence that proves the owner alive after the probe found it gone', async () => { - await seed({ - kind: 'pid-absent', - detail: 'gone', - observedAt: 90_000, - lastProvenAliveAt: 90_001 - }) + it.each([ + ['proves the owner alive after the probe found it gone', { lastProvenAliveAt: 90_001 }], + ['names no possible owner', { ownerFence: -1 }] + ])('quarantines evidence that %s', async (_why, field) => { + await seed({ kind: 'pid-absent', detail: 'gone', observedAt: 90_000, ...field }) expect((await open()).isSessionUnreadable(SESSION)).toBe(true) }) }) diff --git a/src/main/runtime/agent-session-surface-release-transition.ts b/src/main/runtime/agent-session-surface-release-transition.ts index 5c57c18ba41d..eed1c68abb38 100644 --- a/src/main/runtime/agent-session-surface-release-transition.ts +++ b/src/main/runtime/agent-session-surface-release-transition.ts @@ -50,7 +50,8 @@ export function releaseAgentSessionOwnerAfterSurfaceClose(args: { deathEvidence: { kind: 'exit-observed', detail: args.exitReason ?? 'the last surface holding this session released it', - observedAt: args.exitObservedAt ?? args.now + observedAt: args.exitObservedAt ?? args.now, + ownerFence: record.lease.runtimeFence } }) } diff --git a/src/shared/agent-session-lease-adjudication.test.ts b/src/shared/agent-session-lease-adjudication.test.ts index 851eda08281d..f17838e4da79 100644 --- a/src/shared/agent-session-lease-adjudication.test.ts +++ b/src/shared/agent-session-lease-adjudication.test.ts @@ -213,6 +213,7 @@ describe('restart reconciliation', () => { kind: 'identity-mismatch', detail: 'mismatched process-start-time', observedAt: 9_000, + ownerFence: 7, lastProvenAliveAt: 500 } }) @@ -272,6 +273,7 @@ describe('restart reconciliation', () => { kind: 'pid-absent', detail: 'recorded pid absent on host', observedAt: 9_000, + ownerFence: 7, lastProvenAliveAt: 500 } }) @@ -312,7 +314,13 @@ describe('restart reconciliation', () => { ).toEqual({ disposition: 'evicted', nextFence: 8, - evidence: { kind: 'pid-absent', detail: 'reservation never spawned', observedAt: 9_000 } + // The proof is about this reservation's own fence. + evidence: { + kind: 'pid-absent', + detail: 'reservation never spawned', + observedAt: 9_000, + ownerFence: 7 + } }) } ) diff --git a/src/shared/agent-session-lease-adjudication.ts b/src/shared/agent-session-lease-adjudication.ts index d29d5f68d7ce..31ba02949fb9 100644 --- a/src/shared/agent-session-lease-adjudication.ts +++ b/src/shared/agent-session-lease-adjudication.ts @@ -75,7 +75,11 @@ function deathEvidenceFor( lease: AgentSessionLease ): AgentSessionDeathEvidence | null { // Capped so a clock that stepped back across the restart still writes a valid interval. - const interval = { observedAt, lastProvenAliveAt: Math.min(lease.lastRenewedAt, observedAt) } + const interval = { + observedAt, + ownerFence: lease.runtimeFence, + lastProvenAliveAt: Math.min(lease.lastRenewedAt, observedAt) + } if (probe.outcome === 'exit-observed') { return { kind: 'exit-observed', detail: 'observed process exit', ...interval } } @@ -209,7 +213,12 @@ export function adjudicateAgentSessionRestart(args: { nextFence: nextAgentSessionFence(lease), evidence: probe.outcome === 'reservation-unused' - ? { kind: 'pid-absent', detail: 'reservation never spawned', observedAt } + ? { + kind: 'pid-absent', + detail: 'reservation never spawned', + observedAt, + ownerFence: lease.runtimeFence + } : null } } diff --git a/src/shared/agent-session-next-fence.ts b/src/shared/agent-session-next-fence.ts index d5b2f2a08b3b..2c05aa5004db 100644 --- a/src/shared/agent-session-next-fence.ts +++ b/src/shared/agent-session-next-fence.ts @@ -12,8 +12,6 @@ import type { AgentSessionLease } from './agent-session-record' -export function nextAgentSessionFence( - lease: Pick -): number { +export function nextAgentSessionFence(lease: AgentSessionLease): number { return Math.max(lease.runtimeFence + 1, lease.minimumNextFence ?? 0) } diff --git a/src/shared/agent-session-record.ts b/src/shared/agent-session-record.ts index 7487c5944c71..12a26e64ac77 100644 --- a/src/shared/agent-session-record.ts +++ b/src/shared/agent-session-record.ts @@ -87,6 +87,9 @@ export type AgentSessionDeathEvidence = { kind: 'exit-observed' | 'pid-absent' | 'identity-mismatch' detail: string observedAt: number + /** Fence of the owner (or reservation) this death is about; a fence names exactly one. Absent on + * evidence older builds wrote, which then speaks for no turn. */ + ownerFence?: number /** The death interval's lower bound: the last time the runtime holding the owner's transport * proved it alive. Only a probe's proof records it: absent on a surface-release exit, a failed * start, and evidence older builds wrote. */ @@ -282,7 +285,7 @@ function isAgentSessionDeathEvidence(value: unknown): value is AgentSessionDeath return false } const evidence = value as Partial - const { observedAt, lastProvenAliveAt } = evidence + const { observedAt, lastProvenAliveAt, ownerFence } = evidence return ( (evidence.kind === 'exit-observed' || evidence.kind === 'pid-absent' || @@ -291,6 +294,7 @@ function isAgentSessionDeathEvidence(value: unknown): value is AgentSessionDeath typeof observedAt === 'number' && Number.isSafeInteger(observedAt) && observedAt >= 0 && + (ownerFence === undefined || (Number.isSafeInteger(ownerFence) && ownerFence >= 0)) && (lastProvenAliveAt === undefined || (Number.isSafeInteger(lastProvenAliveAt) && lastProvenAliveAt >= 0 && diff --git a/tests/e2e/cross-version-wire/agent-session-death-evidence-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-death-evidence-downgrade.unit.test.ts new file mode 100644 index 000000000000..96780a185176 --- /dev/null +++ b/tests/e2e/cross-version-wire/agent-session-death-evidence-downgrade.unit.test.ts @@ -0,0 +1,61 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, test } from 'vitest' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../../src/shared/agent-session-record.test-fixture' +import { AgentSessionRecordStore } from '../../../src/main/runtime/agent-session-record-store' +import { agentSessionStorePath } from '../../../src/main/runtime/agent-session-record-store-file' +import { importReleaseCheckoutModule, materializeReleaseCheckout } from './release-checkout' + +// The last release before a proof of death named its owner and its last proof of life. +const BASELINE_REF = 'v1.4.211' +const SESSION = 'session-alpha-1' + +test('an older build loads a proof of death that names its owner and its last proof of life', async () => { + const directory = mkdtempSync(join(tmpdir(), 'orca-death-evidence-downgrade-')) + try { + // A live owner at fence 7, last renewed at 30000, found gone when this build relaunches. + writeFileSync( + agentSessionStorePath(directory), + JSON.stringify({ + schemaVersion: 2, + hostId: 'local', + records: { [SESSION]: agentSessionRecordFixture(agentSessionLeaseFixture()) }, + operations: {}, + retiredClaimKeys: [], + unusableRecords: {} + }) + ) + const store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) + await store.reconcileOnRestart({ probe: async () => ({ outcome: 'pid-absent' }), now: 90_000 }) + const evidence = store.getRecord(SESSION)?.lease.deathEvidence + expect(evidence).toEqual({ + kind: 'pid-absent', + detail: 'recorded pid absent on host', + observedAt: 90_000, + ownerFence: 7, + lastProvenAliveAt: 30_000 + }) + + const checkout = await materializeReleaseCheckout(BASELINE_REF) + const baseline = await importReleaseCheckoutModule( + checkout, + 'src/main/runtime/agent-session-record-store.ts' + ) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pinned release exports this class with the open/read members called below; a missing one fails the test. + const OldStore = baseline.AgentSessionRecordStore as { + open: (args: { directory: string; hostId: string }) => Promise + } + const old = await OldStore.open({ directory, hostId: 'local' }) + expect(old.isSessionUnreadable(SESSION)).toBe(false) + expect(old.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'released', + deathEvidence: evidence + }) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) From 1784860b272094132d084b433b3d74e09c137c48 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:03:06 -0700 Subject: [PATCH 14/24] test(native-chat): every writer of a watched exit names the owner it released A watched exit that names no owner reads the older rule, so the settle alone cannot tell a dropped field; the writers are pinned directly, including past a recovery floor. --- ...session-death-evidence-persistence.test.ts | 93 +++++++++++++------ ...session-surface-release-transition.test.ts | 2 + 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/src/main/runtime/agent-session-death-evidence-persistence.test.ts b/src/main/runtime/agent-session-death-evidence-persistence.test.ts index 1f17a1f40bbb..60adbc646eca 100644 --- a/src/main/runtime/agent-session-death-evidence-persistence.test.ts +++ b/src/main/runtime/agent-session-death-evidence-persistence.test.ts @@ -88,7 +88,7 @@ describe('death evidence on disk', () => { }) }) -describe('a failed acquisition parked in recovery', () => { +describe('a failed acquisition', () => { const NOW = 1_800_000_000_000 const OPERATION_ID = `${NOW}-${'1'.padStart(32, '0')}` @@ -133,37 +133,70 @@ describe('a failed acquisition parked in recovery', () => { it.each([ ['before the owner proved its handle', false], ['after the owner proved its handle', true] - ] as const)('keeps the last proof of life, not the failure, %s', async (_when, proved) => { - const store = await open() - const fence = await spawnedOwner(store) - if (proved) { - await store.proveOwner({ + ] as const)( + 'parks in recovery keeping the last proof of life, not the failure, %s', + async (_when, proved) => { + const store = await open() + const fence = await spawnedOwner(store) + if (proved) { + await store.proveOwner({ + sessionId: SESSION, + fence, + link: { + linkId: 'link-1', + handle: { provider: 'claude', sessionId: 'provider-session-1', leafUuid: null }, + origin: 'created', + mintedAtFence: fence, + observedAt: NOW + }, + now: NOW + }) + await store.settleFailedPostAcquisitionAttachment(unproven(fence, NOW + 60_000)) + } else { + await store.settleFailedAcquisition(unproven(fence, NOW + 60_000)) + } + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + handoffStage: 'recovering', + lastRenewedAt: NOW + }) + + const evicted = await store.evictProvenDeadOwner({ sessionId: SESSION, - fence, - link: { - linkId: 'link-1', - handle: { provider: 'claude', sessionId: 'provider-session-1', leafUuid: null }, - origin: 'created', - mintedAtFence: fence, - observedAt: NOW - }, - now: NOW + expectedFence: fence, + probe: { outcome: 'pid-absent' }, + now: NOW + 120_000 }) - await store.settleFailedPostAcquisitionAttachment(unproven(fence, NOW + 60_000)) - } else { - await store.settleFailedAcquisition(unproven(fence, NOW + 60_000)) + expect(evicted.lease.deathEvidence).toMatchObject({ lastProvenAliveAt: NOW }) } - expect(store.getRecord(SESSION)?.lease).toMatchObject({ - handoffStage: 'recovering', - lastRenewedAt: NOW - }) + ) - const evicted = await store.evictProvenDeadOwner({ - sessionId: SESSION, - expectedFence: fence, - probe: { outcome: 'pid-absent' }, - now: NOW + 120_000 - }) - expect(evicted.lease.deathEvidence).toMatchObject({ lastProvenAliveAt: NOW }) - }) + it.each(['exit-proven', 'root-exit-observed', 'processless'] as const)( + 'names its own fence in the proof when cleanup settles it %s', + async (exitProof) => { + const store = await open() + const fence = await spawnedOwner(store) + const settled = + exitProof === 'processless' + ? await store.settleFailedAcquisition({ ...unproven(fence, NOW), exitProof }) + : await (async () => { + await store.proveOwner({ + sessionId: SESSION, + fence, + link: { + linkId: 'link-1', + handle: { provider: 'claude', sessionId: 'provider-session-1', leafUuid: null }, + origin: 'created', + mintedAtFence: fence, + observedAt: NOW + }, + now: NOW + }) + return store.settleFailedPostAcquisitionAttachment({ + ...unproven(fence, NOW), + exitProof + }) + })() + expect(settled.lease.deathEvidence).toMatchObject({ ownerFence: fence }) + } + ) }) diff --git a/src/main/runtime/agent-session-surface-release-transition.test.ts b/src/main/runtime/agent-session-surface-release-transition.test.ts index 5e671acc77f5..0ea30d901976 100644 --- a/src/main/runtime/agent-session-surface-release-transition.test.ts +++ b/src/main/runtime/agent-session-surface-release-transition.test.ts @@ -18,5 +18,7 @@ describe('agent session surface release transition', () => { }) expect(released.lease.runtimeFence).toBe(9) + // The proof names the owner it released, not the fence the floor moved to. + expect(released.lease.deathEvidence).toMatchObject({ kind: 'exit-observed', ownerFence: 7 }) }) }) From 7ff03e3742d220381f9ccb227efa584e5035cede Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:06:31 -0700 Subject: [PATCH 15/24] test(native-chat): give the fake journal's cast its safety rationale --- .../structured-agent-session-stale-turn-verdict.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 05c34904a694..a392499f2435 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -283,6 +283,7 @@ describe('running turn lifecycle revisions', () => { describe('stale session state on a cold acquire', () => { function journalWith(items: AgentJournalRenderItem[]) { const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 9 })) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the settle reads only these journal members, and a release that proved nothing never reads the live bound. const journal = { snapshot: () => ({ items }), itemFence: () => 1, From 45b6d97a01b6baebc8181a3db06462aa83812bb3 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:17:43 -0700 Subject: [PATCH 16/24] fix(native-chat): a proof of death written after a chat opened revises the turn it left unverifiable On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner, so the open settles the cut-off turn unverifiable. When the reconcile then records the proof, it re-runs the same settle for every open conversation, which revises that owner's unverifiable turns to interrupted with the proof's end. Any later open re-runs it too, so a failed write converges. Only upward, only for a proof that names the turn's own owner. --- ...uctured-agent-session-conversation-open.ts | 57 +++++++++++++------ ...gent-session-dead-generation-settlement.ts | 15 +++-- .../structured-agent-session-host-delivery.ts | 13 +++++ .../structured-agent-session-host.ts | 4 +- ...uctured-agent-session-restart-reconcile.ts | 4 +- ...ctured-agent-session-stale-turn-verdict.ts | 56 ++++++++++++------ 6 files changed, 111 insertions(+), 38 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index 07d1fb6b5ffb..7d13600fe7dd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -10,6 +10,7 @@ import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecord } from '../../../shared/agent-session-record' @@ -102,22 +103,11 @@ export async function openStructuredAgentSessionConversationJournal( } catch (error) { deps.onEventSinkError?.({ sessionId, error }) } - try { - // No child in this process writes to a journal nobody had open, so whatever it shows running - // belongs to a generation that is gone, whatever the lease still claims. Settled before any - // reader or child sees it. - if (!options.acquisition) { - await settleStaleStructuredAgentSessionState({ - journal: opened.journal, - sessionId, - fence, - acquisitionGeneration: null, - deathEvidence: record.lease.deathEvidence ?? null - }) - } - } catch (error) { - // Best effort: the next acquire re-derives it. - deps.onEventSinkError?.({ sessionId, error }) + // No child in this process writes to a journal nobody had open, so whatever it shows running + // belongs to a generation that is gone, whatever the lease still claims. Settled before any + // reader or child sees it. + if (!options.acquisition) { + await settleGoneGeneration(deps, record, opened.journal) } return { session: { journal: opened.journal, params, child: null }, @@ -125,6 +115,41 @@ export async function openStructuredAgentSessionConversationJournal( } } +/** + * The open's settle again, for a conversation already open: a proof of death written since it + * opened (the startup reconcile) revises what the open could only call `unverifiable`. A record + * holds a proof only while released, so no child here is writing. A no-op once it has revised. + */ +export async function resettleOpenStructuredAgentSessionConversation( + deps: StructuredAgentSessionConversationOpenDeps, + sessionId: string, + session: StructuredAgentSessionHostSession | undefined +): Promise { + const record = deps.store.getRecord(sessionId) + if (session && record?.lease.deathEvidence) { + await settleGoneGeneration(deps, record, session.journal) + } +} + +async function settleGoneGeneration( + deps: Pick, + record: AgentSessionRecord, + journal: AgentSessionJournal +): Promise { + try { + await settleStaleStructuredAgentSessionState({ + journal, + sessionId: record.sessionId, + fence: record.lease.runtimeFence, + acquisitionGeneration: null, + deathEvidence: record.lease.deathEvidence ?? null + }) + } catch (error) { + // Best effort: the next open or acquire re-derives it. + deps.onEventSinkError?.({ sessionId: record.sessionId, error }) + } +} + export function attachParamsForRecord( record: AgentSessionRecord, input: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index c3a90842273b..fd72122d357c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -18,6 +18,7 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store import { structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { + provenUnverifiableTurnRevisions, runningTurnLifecycleRevisions, turnVerdictFromDeathEvidence, type StructuredAgentSessionTurnVerdict @@ -231,8 +232,9 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { * Settles whatever a generation with no child in this process left running: found when a new child * is acquired, or when a chat is reopened for reading. Derived from the journal and the lease's * death evidence each time, so nothing is owed in between. Proven death ends the turn interrupted, - * and only a watched exit carries its reason into the copy. Must run before a new child's buffered - * events land, or a live turn would be judged. + * and only a watched exit carries its reason into the copy; a proof written after an earlier settle + * revises what that settle left `unverifiable`. Must run before a new child's buffered events land, + * or a live turn would be judged. */ export async function settleStaleStructuredAgentSessionState(input: { journal: AgentSessionJournal @@ -256,10 +258,15 @@ export async function settleStaleStructuredAgentSessionState(input: { mutations.push({ kind: 'item', identity, body }) } } + const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal) mutations.push( - ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))) + ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), + ...proven ) - if (items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted')) { + if ( + proven.length > 0 || + items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted') + ) { mutations.unshift({ kind: 'item', identity: { provider: 'orca', clientMessageId: settlementId }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index 63d0b0143a43..f6aef68d9d44 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -7,6 +7,7 @@ import type { AgentJournalResetReason } from '../../../shared/agent-session-jour import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openStructuredAgentSessionConversation, + resettleOpenStructuredAgentSessionConversation, type OpenedStructuredAgentSessionConversation, type StructuredAgentSessionConversationOpenOptions } from './structured-agent-session-conversation-open' @@ -28,6 +29,8 @@ export type StructuredAgentSessionConversationDelivery = { sessionId: string, options?: StructuredAgentSessionConversationOpenOptions ) => Promise + /** Re-runs the open's settle for every open conversation no child here drives. */ + resettleOpen: () => void /** Indexes a conversation some other open produced, as `open` would have. */ adoptOpened: ( sessionId: string, @@ -79,6 +82,16 @@ export function createStructuredAgentSessionConversationDelivery(input: { return { loop, adoptOpened, + // Queued, never awaited: an attach holding a session's serialize waits on the reconcile. + resettleOpen: () => { + for (const sessionId of sessions.keys()) { + void input + .serialize(sessionId, () => + resettleOpenStructuredAgentSessionConversation(deps, sessionId, sessions.get(sessionId)) + ) + .catch((error: unknown) => deps.onEventSinkError?.({ sessionId, error })) + } + }, open: (sessionId, options) => openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index c1a3075cbee5..c1a380511f26 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -106,7 +106,9 @@ export class StructuredAgentSessionHost { store: deps.store, probe: (record) => this.runtimeState.probeRecord(record), ...(deps.probeOwners ? { probeMany: deps.probeOwners } : {}), - now: () => this.now() + now: () => this.now(), + // A read can open a chat before the reconcile proves its owner dead; this revises its turn. + onReconciled: () => this.conversationDelivery.resettleOpen() }) this.conversationDelivery = createStructuredAgentSessionConversationDelivery({ deps, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index 3b6cc30271e5..d5be516f1c82 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -20,6 +20,8 @@ export function createRestartReconciler(deps: { records: readonly AgentSessionRecord[] ) => Promise> now: () => number + /** Told once a run has written what it proved, even a run that then gave up. Must not throw. */ + onReconciled?: () => void }): (sessionId: string) => Promise { let pending: Promise | null = null return async (sessionId) => { @@ -27,7 +29,7 @@ export function createRestartReconciler(deps: { return null } if (!pending) { - const run = reconcileCurrentLeases(deps) + const run = reconcileCurrentLeases(deps).finally(() => deps.onReconciled?.()) pending = run.finally(() => { pending = null }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 851b09d30209..8649b0083cb4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -4,7 +4,7 @@ // that turn's owner by fence — a watched exit, or a local probe that found the recorded pid gone or // reused. A release nothing proved — lost contact, an unverifiable identity, a stop that outlived the // ladder — carries none, and neither does a later owner's death; the turn is then `unverifiable` -// with no end at all. +// with no end at all, until a proof naming its owner is written and revises it. import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { @@ -62,23 +62,47 @@ export function runningTurnLifecycleRevisions( items: readonly AgentJournalRenderItem[], verdict: StructuredAgentSessionTurnVerdict ): JournalLifecycleMutationInput[] { - const revisions: JournalLifecycleMutationInput[] = [] - for (const item of items) { + return items.flatMap((item) => { const turn = readAgentJournalTurn(item.body) - if (turn?.state !== 'running') { - continue - } - const identity = parseAgentJournalItemKey(item.itemId) - if (!identity) { - continue - } - revisions.push({ - kind: 'item', - identity, - body: agentJournalTurnBody(settledLifecycle(turn, verdict)) - }) + return turn?.state === 'running' ? turnLifecycleRevision(item, turn, verdict) : [] + }) +} + +/** + * A turn an earlier settle could only call `unverifiable`, because the proof had not been written + * yet, revised once a proof names the owner that wrote it. Only ever upward, and never from an + * older build's proof, which names no owner. + */ +export function provenUnverifiableTurnRevisions( + items: readonly AgentJournalRenderItem[], + evidence: AgentSessionDeathEvidence | null | undefined, + journal: Pick +): JournalLifecycleMutationInput[] { + const ownerFence = evidence?.ownerFence + if (ownerFence === undefined) { + return [] } - return revisions + return items.flatMap((item) => { + const turn = readAgentJournalTurn(item.body) + return turn?.state === 'unverifiable' && journal.itemFence(item.itemId) === ownerFence + ? turnLifecycleRevision( + item, + turn, + turnVerdictFromDeathEvidence(evidence, journal, ownerFence) + ) + : [] + }) +} + +function turnLifecycleRevision( + item: AgentJournalRenderItem, + turn: AgentJournalTurnLifecycle, + verdict: StructuredAgentSessionTurnVerdict +): JournalLifecycleMutationInput[] { + const identity = parseAgentJournalItemKey(item.itemId) + return identity + ? [{ kind: 'item', identity, body: agentJournalTurnBody(settledLifecycle(turn, verdict)) }] + : [] } /** The verdict owns the turn's end and nothing else; every other field the row From db09dd4e0222bead8e8f246866ecc23b17f9c946 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:17:53 -0700 Subject: [PATCH 17/24] test(native-chat): a chat read before the reconcile reads unverifiable, then interrupted Covers the reconcile revising an open chat to the last renewal (27 s) and a subscriber being sent both states, a start after the crash whose running turn the queued revision leaves alone, a failed revision write converging at the next open, a proof about another owner or from an older build never revising, and a second settle writing nothing. --- ...tured-agent-session-crash-turn-end.test.ts | 165 ++++++++++++++++-- ...agent-session-send-open-stale-turn.test.ts | 8 +- ...d-agent-session-stale-turn-verdict.test.ts | 68 ++++++++ 3 files changed, 228 insertions(+), 13 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index ea10b320dfcb..5a6af0d22d13 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -10,6 +10,7 @@ import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { completedStructuredAgentTurnSeconds, @@ -24,6 +25,7 @@ import { type StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idle-sweep' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { hostTestAttachParams, @@ -233,6 +235,158 @@ describe('a turn a crash cut short mid-tool', () => { }) }) +/** Every turn state a subscriber was sent, in order: snapshots, then live batches. */ +function turnStatesSent(events: readonly AgentSessionSubscribeEvent[]) { + return events + .flatMap((event) => + event.type === 'snapshot' ? event.page.items : event.type === 'batch' ? event.batch.items : [] + ) + .flatMap((item) => readAgentJournalTurn(item.body) ?? []) +} + +function attach(fence: number) { + return host.attach( + { callerKey: 'client-1' }, + hostTestAttachParams(fence, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null } + }) + ) +} + +/** Runs whatever this session's serialize already has queued. */ +function drainSession(): Promise { + return host.collaboratorsForTests().serialize(SESSION, async () => {}) +} + +// On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner, +// so the open can only call the turn unverifiable; the reconcile's proof then revises it. +describe('a turn a read reached before the reconcile proved its owner dead', () => { + it('reads unverifiable, then interrupted at the last renewal, and a subscriber is sent both', async () => { + openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) + const events: AgentSessionSubscribeEvent[] = [] + const unsubscribe = await host.subscribe({ + id: 'reader-1', + sessionId: SESSION, + emit: (event) => events.push(event) + }) + expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + + await host.reconcileRestartLeases() + await drainSession() + + // A paired or phone client learns of the revision through the ordinary journal publish. + await vi.waitFor(() => + expect(turnStatesSent(events)).toMatchObject([ + { state: 'unverifiable' }, + { state: 'interrupted', completedAt: LAST_RENEWED_AT } + ]) + ) + const { items } = await host.journalSnapshot(SESSION) + const [timing] = selectStructuredAgentTurnTimings(items).values() + expect(completedStructuredAgentTurnSeconds(timing)).toBe(27) + expect(items.filter((item) => item.body.kind === 'status')).toHaveLength(1) + unsubscribe() + }) + + it('revises nothing twice, whoever re-runs the settle', async () => { + openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) + await host.history({ sessionId: SESSION, direction: 'tail' }) + await host.reconcileRestartLeases() + await drainSession() + const settled = await host.journalSnapshot(SESSION) + + host.collaboratorsForTests().conversationDelivery.resettleOpen() + await drainSession() + await host.restoreReadableSessions() + + expect(await host.journalSnapshot(SESSION)).toEqual(settled) + }) + + it('never touches the turn a start after the crash is writing', async () => { + const acquire = vi.fn( + async ({ fence, spawnToken, onSpawned, events }) => { + const process = { + hostId: 'local', + pid: 8_000, + processStartTimeMs: RELAUNCHED_AT, + spawnToken + } + await onSpawned?.(process) + // The new child is already working when its start lands, ahead of the queued revision. + events?.appendItem( + { provider: 'claude', sessionId: PROVIDER_SESSION, uuid: 'uuid-turn-2' }, + { kind: 'turn', turnId: 'turn-2', state: 'running', startedAt: RELAUNCHED_AT } + ) + return { + process, + link: { + linkId: `claude-${fence}-link`, + handle: { provider: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null }, + origin: 'resumed', + mintedAtFence: fence, + observedAt: RELAUNCHED_AT + } + } + } + ) + openHost({ + adapter: { + acquire, + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + await host.history({ sessionId: SESSION, direction: 'tail' }) + + // The start runs the reconcile itself, so the revision it queues waits behind the start. + await expect(attach(14)).resolves.toMatchObject({ ok: true }) + await host.flushStreamedEvents(SESSION) + await drainSession() + + const turns = (await host.journalSnapshot(SESSION)).items.flatMap( + (item) => readAgentJournalTurn(item.body) ?? [] + ) + expect(turns).toMatchObject([ + { turnId: 'turn-1', state: 'interrupted', completedAt: LAST_RENEWED_AT }, + { turnId: 'turn-2', state: 'running' } + ]) + }) + + it('stays unverifiable when the revision cannot be written, and a later open revises it', async () => { + let now = RELAUNCHED_AT + const onEventSinkError = vi.fn() + openHost({ + probeOwner: async () => ({ outcome: 'pid-absent' }), + now: () => now, + onEventSinkError + }) + await host.history({ sessionId: SESSION, direction: 'tail' }) + const { journal } = host.collaboratorsForTests().sessions.get(SESSION)! + vi.spyOn(journal, 'appendLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) + + await host.reconcileRestartLeases() + await drainSession() + + expect(onEventSinkError).toHaveBeenCalledOnce() + expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + // The proof is durable on the record, so the next open converges. + now += STRUCTURED_AGENT_SESSION_IDLE_MS + 1 + await host.collaboratorsForTests().lifetime.idleSweep.tick() + expect(host.hasSession(SESSION)).toBe(false) + expect(await settledTurn()).toMatchObject({ + state: 'interrupted', + completedAt: LAST_RENEWED_AT + }) + }) +}) + /** A relaunched host whose first start after the crash, at fence 15, fails with `failure`; every * later start succeeds. The crashed owner, and any child left behind, probe gone. */ async function hostWithFailingFirstStart(failure: Error) { @@ -271,16 +425,7 @@ async function hostWithFailingFirstStart(failure: Error) { }) await host.reconcileRestartLeases() return { - attach: (fence: number) => - host.attach( - { callerKey: 'client-1' }, - hostTestAttachParams(fence, { - provider: 'claude', - agent: 'claude', - accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, - providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION, leafUuid: null } - }) - ), + attach, advance: (ms: number) => { now += ms } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts index 23bcde573b3b..0a14f4361197 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -141,15 +141,17 @@ it.each(PROBES)( } ) -// On desktop the chat on screen at relaunch reads before startup reconciles the leases. -it('settles it when a read reaches the chat before the startup reconcile', async () => { +// On desktop the chat on screen at relaunch reads before startup reconciles the leases: nothing +// has proved its owner gone yet, and the reconcile's proof then revises what the open settled. +it('settles it when a read reaches the chat before the startup reconcile, then revises it', async () => { const { host } = await relaunchAfterCrashMidTurn({ outcome: 'pid-absent' }, { reconcile: false }) expect(hostTestState().store.getRecord(SESSION)?.lease.claimStatus).toBe('live') await host.history({ sessionId: SESSION, direction: 'tail' }) + expect(await turnStates(host)).toEqual(['unverifiable']) await host.reconcileRestartLeases() await host.restoreReadableSessions([SESSION]) - expect(await turnStates(host)).toEqual(['unverifiable']) + expect(await turnStates(host)).toEqual(['interrupted']) await host.flushAllStreamedEvents() }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index a392499f2435..57f2fbe1a269 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -280,6 +280,14 @@ describe('running turn lifecycle revisions', () => { }) }) +/** A probe that found the fence-1 owner gone at 9000. */ +const PROVEN: AgentSessionDeathEvidence = { + kind: 'pid-absent', + detail: 'recorded pid absent on host', + observedAt: 9_000, + ownerFence: 1 +} + describe('stale session state on a cold acquire', () => { function journalWith(items: AgentJournalRenderItem[]) { const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 9 })) @@ -540,6 +548,66 @@ describe('stale session state on a cold acquire', () => { } }) + it('revises an unverifiable turn only from a proof naming its own owner, and only once', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) + const journals = createTrackedJournalOpener() + let now = 100 + try { + const journal = await journals.open({ + identity: { + sessionId: 'session-1', + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: THREAD } + }, + journalDir: root, + now: () => now + }) + await journal.appendItem( + RUNNING_IDENTITY, + { kind: 'turn', turnId: 'turn-2', state: 'running', startedAt: 100 }, + { fence: 1 } + ) + now = 400 + // The open, before anything proved the fence-1 owner gone. + const settle = (deathEvidence: AgentSessionDeathEvidence | null) => + settleStaleStructuredAgentSessionState({ + journal, + sessionId: 'session-1', + fence: 2, + acquisitionGeneration: null, + deathEvidence + }) + await settle(null) + now = 9_000 + const turn = () => journal.snapshot().items.map((item) => readAgentJournalTurn(item.body))[0] + expect(turn()).toMatchObject({ state: 'unverifiable' }) + + const unrevised = journal.cursor() + await expect(settle({ ...PROVEN, ownerFence: 2 })).resolves.toBe(0) + const { ownerFence: _ownerFence, ...olderBuildProof } = PROVEN + await expect(settle({ ...olderBuildProof, kind: 'exit-observed' })).resolves.toBe(0) + expect(journal.cursor()).toEqual(unrevised) + expect(turn()).toMatchObject({ state: 'unverifiable' }) + + await expect(settle(PROVEN)).resolves.toBe(2) + expect(turn()).toEqual({ + turnId: 'turn-2', + state: 'interrupted', + startedAt: 100, + completedAt: 100 + }) + const revised = journal.cursor() + await expect(settle(PROVEN)).resolves.toBe(0) + expect(journal.cursor()).toEqual(revised) + expect(journal.snapshot().items.filter((item) => item.body.kind === 'status')).toHaveLength(1) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + }) + it('writes nothing when no turn is running and keys on the journal position without a generation', async () => { const idle = journalWith([ lifecycleItem('turn-1', 'completed', 1, { startedAt: 10, completedAt: 20 }) From 54a4dd14fcbfc3f1febfa5cce927bc4fe338d08f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:56:02 -0700 Subject: [PATCH 18/24] fix(native-chat): revise an open chat's turn wherever a proof of death is written The store tells its listeners, once committed, of each record a transaction gave a new proof of death, so every writer (the startup reconcile, a recovery that stopped a child which outlived Orca, a failed start, a watched exit) triggers the same serialized resettle for a chat already open. The reconcile's own callback is gone. Quit stops listening first, and a queued resettle is drained with the starts. --- ...uctured-agent-session-conversation-open.ts | 4 +- ...tured-agent-session-crash-turn-end.test.ts | 51 +++++++++++++++++-- .../structured-agent-session-host-delivery.ts | 29 +++++++---- .../structured-agent-session-host.ts | 6 +-- ...uctured-agent-session-restart-reconcile.ts | 4 +- ...session-death-evidence-persistence.test.ts | 28 +++++++++- .../runtime/agent-session-record-store.ts | 30 ++++++++++- 7 files changed, 127 insertions(+), 25 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index 7d13600fe7dd..d6c83d688b3c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -117,8 +117,8 @@ export async function openStructuredAgentSessionConversationJournal( /** * The open's settle again, for a conversation already open: a proof of death written since it - * opened (the startup reconcile) revises what the open could only call `unverifiable`. A record - * holds a proof only while released, so no child here is writing. A no-op once it has revised. + * opened (the startup reconcile, a recovery) revises what the open could only call `unverifiable`. + * A record holds a proof only while released, so no child here is writing. A no-op once revised. */ export async function resettleOpenStructuredAgentSessionConversation( deps: StructuredAgentSessionConversationOpenDeps, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index 5a6af0d22d13..0d83491ee21f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -24,6 +24,7 @@ import { AgentSessionAcquisitionExitUnprovenError, type StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { resettleOpenStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idle-sweep' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' @@ -262,7 +263,7 @@ function drainSession(): Promise { } // On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner, -// so the open can only call the turn unverifiable; the reconcile's proof then revises it. +// so the open can only call the turn unverifiable; the proof, whoever writes it, then revises it. describe('a turn a read reached before the reconcile proved its owner dead', () => { it('reads unverifiable, then interrupted at the last renewal, and a subscriber is sent both', async () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) @@ -298,13 +299,57 @@ describe('a turn a read reached before the reconcile proved its owner dead', () await drainSession() const settled = await host.journalSnapshot(SESSION) - host.collaboratorsForTests().conversationDelivery.resettleOpen() - await drainSession() + await host + .collaboratorsForTests() + .serialize(SESSION, () => + resettleOpenStructuredAgentSessionConversation( + host.deps, + SESSION, + host.collaboratorsForTests().sessions.get(SESSION) + ) + ) await host.restoreReadableSessions() expect(await host.journalSnapshot(SESSION)).toEqual(settled) }) + it('revises it when recovery stops a child that outlived Orca, with no send', async () => { + let alive = true + const probeOwner = async (): Promise => + alive + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + const acquire = vi.fn() + openHost({ + adapter: { + acquire, + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + probeOwner, + stopOwnerProcess + }) + await host.history({ sessionId: SESSION, direction: 'tail' }) + expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + + // The reconcile only parks the live orphan in recovery; recovery's stop is what proves it gone. + await host.restoreReadableSessions() + await drainSession() + + expect(stopOwnerProcess).toHaveBeenCalledOnce() + expect(acquire).not.toHaveBeenCalled() + expect(await settledTurn()).toMatchObject({ + state: 'interrupted', + completedAt: LAST_RENEWED_AT + }) + }) + it('never touches the turn a start after the crash is writing', async () => { const acquire = vi.fn( async ({ fence, spawnToken, onSpawned, events }) => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index f6aef68d9d44..6f2e1e41d94e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -29,8 +29,8 @@ export type StructuredAgentSessionConversationDelivery = { sessionId: string, options?: StructuredAgentSessionConversationOpenOptions ) => Promise - /** Re-runs the open's settle for every open conversation no child here drives. */ - resettleOpen: () => void + /** Stops the loop and the resettle on a proof of death; quit's first step. */ + dispose: () => void /** Indexes a conversation some other open produced, as `open` would have. */ adoptOpened: ( sessionId: string, @@ -79,18 +79,25 @@ export function createStructuredAgentSessionConversationDelivery(input: { loop.wake(sessionId) } } + // A chat open before its owner's death was proven revises what its open settled. Queued, never + // awaited: the writer can hold this session's serialize (an attach recovering its lease). + const stopResettling = deps.store.onDeathEvidence((sessionId) => { + if (sessions.has(sessionId)) { + void input + .trackStart( + input.serialize(sessionId, () => + resettleOpenStructuredAgentSessionConversation(deps, sessionId, sessions.get(sessionId)) + ) + ) + .catch((error: unknown) => deps.onEventSinkError?.({ sessionId, error })) + } + }) return { loop, adoptOpened, - // Queued, never awaited: an attach holding a session's serialize waits on the reconcile. - resettleOpen: () => { - for (const sessionId of sessions.keys()) { - void input - .serialize(sessionId, () => - resettleOpenStructuredAgentSessionConversation(deps, sessionId, sessions.get(sessionId)) - ) - .catch((error: unknown) => deps.onEventSinkError?.({ sessionId, error })) - } + dispose: () => { + loop.dispose() + stopResettling() }, open: (sessionId, options) => openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index c1a380511f26..efd27ad746e0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -106,9 +106,7 @@ export class StructuredAgentSessionHost { store: deps.store, probe: (record) => this.runtimeState.probeRecord(record), ...(deps.probeOwners ? { probeMany: deps.probeOwners } : {}), - now: () => this.now(), - // A read can open a chat before the reconcile proves its owner dead; this revises its turn. - onReconciled: () => this.conversationDelivery.resettleOpen() + now: () => this.now() }) this.conversationDelivery = createStructuredAgentSessionConversationDelivery({ deps, @@ -249,7 +247,7 @@ export class StructuredAgentSessionHost { // Trigger inlined rather than imported: `AgentSessionResumeTrigger` in shared is the canonical // type, and this file has no line budget left for the import. async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise { - this.conversationDelivery.loop.dispose() + this.conversationDelivery.dispose() await flushStructuredAgentSessionHost({ ...this.lifetimeContext(), idleSweep: this.lifetime, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index d5be516f1c82..3b6cc30271e5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -20,8 +20,6 @@ export function createRestartReconciler(deps: { records: readonly AgentSessionRecord[] ) => Promise> now: () => number - /** Told once a run has written what it proved, even a run that then gave up. Must not throw. */ - onReconciled?: () => void }): (sessionId: string) => Promise { let pending: Promise | null = null return async (sessionId) => { @@ -29,7 +27,7 @@ export function createRestartReconciler(deps: { return null } if (!pending) { - const run = reconcileCurrentLeases(deps).finally(() => deps.onReconciled?.()) + const run = reconcileCurrentLeases(deps) pending = run.finally(() => { pending = null }) diff --git a/src/main/runtime/agent-session-death-evidence-persistence.test.ts b/src/main/runtime/agent-session-death-evidence-persistence.test.ts index 60adbc646eca..6e0bff8098cd 100644 --- a/src/main/runtime/agent-session-death-evidence-persistence.test.ts +++ b/src/main/runtime/agent-session-death-evidence-persistence.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionDeathEvidence } from '../../shared/agent-session-record' import { agentSessionLeaseFixture, @@ -88,6 +88,32 @@ describe('death evidence on disk', () => { }) }) +describe('who is told a proof of death landed', () => { + it('tells a listener once the proof is committed, and never for a write that proves nothing', async () => { + await seed(null) + const store = await open() + const told: unknown[] = [] + store.onDeathEvidence((sessionId) => told.push(store.getRecord(sessionId)?.lease.deathEvidence)) + const unsubscribed = vi.fn() + store.onDeathEvidence(unsubscribed)() + + await expect( + store.evictProvenDeadOwner({ + sessionId: SESSION, + expectedFence: 6, + probe: { outcome: 'pid-absent' }, + now: 80_000 + }) + ).rejects.toThrow() + await store.reconcileOnRestart({ probe: async () => ({ outcome: 'pid-absent' }), now: 90_000 }) + // The proof already on the record is not news to a later write. + await store.setSessionTabVisibility(SESSION, true) + + expect(told).toEqual([expect.objectContaining({ kind: 'pid-absent', ownerFence: 7 })]) + expect(unsubscribed).not.toHaveBeenCalled() + }) +}) + describe('a failed acquisition', () => { const NOW = 1_800_000_000_000 const OPERATION_ID = `${NOW}-${'1'.padStart(32, '0')}` diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index b2bf36323015..62d53c17b6de 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -77,6 +77,8 @@ export const AGENT_SESSION_LEASE_TTL_MS = 30_000, AGENT_SESSION_LEASE_RENEW_INTERVAL_MS = 10_000 export class AgentSessionRecordStore { + private readonly deathEvidenceListeners = new Set<(sessionId: string) => void>() + private constructor(private readonly transactions: AgentSessionStoreTransactionQueue) {} static async open(args: { directory: string; hostId: string }): Promise { @@ -338,6 +340,32 @@ export class AgentSessionRecordStore { }) } + /** Told, once committed, of each session a transaction wrote a proof of death for — whichever + * transition wrote it, since every one lands here. Must not throw. */ + onDeathEvidence(listener: (sessionId: string) => void): () => void { + this.deathEvidenceListeners.add(listener) + return () => this.deathEvidenceListeners.delete(listener) + } + /** Serialize every mutation against the latest committed disk state. */ - private transact = (apply: () => T): Promise => this.transactions.transact(apply) + private transact = async (apply: () => T): Promise => { + let proven: string[] = [] + const result = await this.transactions.transact(() => { + if (this.deathEvidenceListeners.size === 0) { + return apply() + } + const before = new Map( + [...this.state.records].map(([id, record]) => [id, record.lease.deathEvidence]) + ) + const applied = apply() + proven = [...this.state.records] + .filter(([id, { lease }]) => lease.deathEvidence && lease.deathEvidence !== before.get(id)) + .map(([id]) => id) + return applied + }) + for (const sessionId of proven) { + this.deathEvidenceListeners.forEach((listener) => listener(sessionId)) + } + return result + } } From e4ce520a105c50a3744fb8517c78ca842bd95f0a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:59:37 -0700 Subject: [PATCH 19/24] test(native-chat): a failed exit settlement is retried in place once the exit is recorded Recording a watched exit now queues the same settle an open runs, so the turn converges without waiting for the chat to be reopened. The reopen and read-after-restart cases now refuse that retry too, so they still pin the open's own settle. --- ...ured-agent-session-stale-turn-open.test.ts | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts index 2134e3392928..41eebd747374 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts @@ -29,8 +29,9 @@ afterEach(async () => { await rig.dispose() }) -/** A turn in flight whose agent exits, and whose exit settlement the journal refuses. */ -async function exitWithUnwrittenSettlement(): Promise { +/** A turn in flight whose agent exits, and whose exit settlement the journal refuses: by default + * also the retry that recording the exit queues, so only an open is left to settle it. */ +async function exitWithUnwrittenSettlement(refusedWrites = 2): Promise { await foundRestTestChat(rig) const open = rig.host.collaboratorsForTests().sessions.get(SESSION)! const running = open.child! @@ -41,7 +42,10 @@ async function exitWithUnwrittenSettlement(): Promise { { kind: 'turn', turnId: 'working', state: 'running' } ) await rig.host.flushStreamedEvents(SESSION) - vi.spyOn(open.journal, 'appendLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) + const append = vi.spyOn(open.journal, 'appendLifecycleBatch') + for (let refused = 0; refused < refusedWrites; refused += 1) { + append.mockRejectedValueOnce(new Error('disk full')) + } await rig.host.handleAdapterEvent({ type: 'ended', sessionId: SESSION, @@ -56,6 +60,7 @@ async function exitWithUnwrittenSettlement(): Promise { deathEvidence: { kind: 'exit-observed' } }) ) + await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) } async function workingTurnState(): Promise { @@ -66,6 +71,16 @@ async function workingTurnState(): Promise { } describe('a turn its gone agent left running', () => { + it('is settled in place by the retry recording the exit queues, with no reopen', async () => { + await exitWithUnwrittenSettlement(1) + + expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(true) + expect(await workingTurnState()).toBe('interrupted') + const { items } = await rig.host.journalSnapshot(SESSION) + expect(JSON.stringify(items)).toContain(unexpectedProviderExitOutcome('killed')) + expect(rig.adapter.acquire).toHaveBeenCalledOnce() + }) + it('is settled when the idle-closed conversation is opened again, and its reader sees it', async () => { await exitWithUnwrittenSettlement() expect(await workingTurnState()).toBe('running') From 1eca100c35b73c8f89d8396d05cd9367ef0b07b5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 02:07:51 -0700 Subject: [PATCH 20/24] test(native-chat): tests that pin a send settling a failed exit refuse the in-place retry too The exit's release now queues the same settle, so two tests named for the send's settle refuse that retry as well; the comments that said nothing retries it now say what does. --- .../structured-agent-session-surface-lifetime.test.ts | 4 +++- .../structured-agent-session-unexpected-exit.test.ts | 2 +- .../structured-agent-session-unexpected-exit.ts | 4 ++-- .../rpc/methods/structured-agent-session-at-rest.test.ts | 6 +++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index 7f4a763e93b6..acc988048ed0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -760,7 +760,9 @@ describe('an unexpected provider exit', () => { runtimeFence: exitedFence + 1, deathEvidence: { kind: 'exit-observed', detail: 'provider exited', observedAt: NOW } }) - // Nothing retries the settlement; the journal writes again, and the next acquire re-derives it. + // The retry recording the exit queues is refused too; once the journal writes again, the next + // acquire re-derives it. + await host.collaboratorsForTests().serialize(SESSION, async () => {}) refusing.mockRestore() dispatch.mockResolvedValueOnce({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 22b6c8fad830..af5eec94bd45 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -124,7 +124,7 @@ describe('provider-exit settlement', () => { observedAt } ) - // Released, not latched: the next acquire or read restore settles what this write left. + // Released, not latched: a later settle from this evidence finishes what this write left. expect(record.lease).toMatchObject({ claimStatus: 'released', handoffStage: null, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts index 834bc219e018..ef3fbb8d3975 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts @@ -127,8 +127,8 @@ export async function settleUnexpectedStructuredAgentSessionExit< acquisitionGeneration: child.generation, now: context.now(), exitObservedAt: observedAt, - // Bare cause: whatever this settlement could not write is settled from it later, by the - // next acquire or read restore, and `exit-observed` already says the rest. + // Bare cause: whatever this settlement could not write is settled from it later (the + // settle recording it queues, or the next open or acquire); `exit-observed` says the rest. exitReason: unexpectedEvent.reason.slice(0, MAX_UNEXPECTED_EXIT_REASON_CHARS) }) } catch (error) { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts index 8202f7932894..724ae7529bd6 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts @@ -314,7 +314,10 @@ describe('an agent exit', () => { { kind: 'turn', turnId: 'working', state: 'running' } ) await rig.host.flushStreamedEvents(SESSION) - vi.spyOn(open.journal, 'appendLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) + // The exit's own write and the retry recording the exit queues are both refused. + vi.spyOn(open.journal, 'appendLifecycleBatch') + .mockRejectedValueOnce(new Error('disk full')) + .mockRejectedValueOnce(new Error('disk full')) await rig.host.handleAdapterEvent({ type: 'ended', sessionId: SESSION, @@ -330,6 +333,7 @@ describe('an agent exit', () => { deathEvidence: { kind: 'exit-observed' } }) ) + await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) const sent = await rig.host.send( CALLER, From 286b1b3f1e1a67a6bcabe69222a28cbdedd18087 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 02:57:44 -0700 Subject: [PATCH 21/24] fix(native-chat): name the explanation row by the death it explains, so a retried settle adds no second row --- ...gent-session-dead-generation-settlement.ts | 13 ++++- ...d-agent-session-stale-turn-verdict.test.ts | 58 +++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index daf4bb1b4a98..2050981f68be 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -204,6 +204,7 @@ export async function settleStaleStructuredAgentSessionState(input: { // Each turn is judged by the evidence only if it names that turn's owner. const verdictFor = (item: AgentJournalRenderItem) => turnVerdictFromDeathEvidence(input.deathEvidence, journal, journal.itemFence(item.itemId)) + // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` const mutations: JournalLifecycleMutationInput[] = [] @@ -219,13 +220,19 @@ export async function settleStaleStructuredAgentSessionState(input: { ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), ...proven ) + const evidence = input.deathEvidence if ( - proven.length > 0 || - items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted') + evidence && + (proven.length > 0 || + items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted')) ) { mutations.unshift({ kind: 'item', - identity: { provider: 'orca', clientMessageId: settlementId }, + // Named by the death it explains, so a retry after a partly written settle adds no second row. + identity: { + provider: 'orca', + clientMessageId: `stale-session:${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` + }, // The death evidence is Orca's log text, never a sentence for a person: the row says only // that the provider stopped. body: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 5a327ccfc7c1..214512605360 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -608,6 +608,64 @@ describe('stale session state on a cold acquire', () => { } }) + it('adds one row for a death however many attempts its settle takes to write', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) + const journals = createTrackedJournalOpener() + let now = 100 + try { + const journal = await journals.open({ + identity: { + sessionId: 'session-1', + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: THREAD } + }, + journalDir: root, + now: () => now + }) + // Enough running turns that the settle writes two batches, and its retry two again. + for (let index = 0; index < 399; index++) { + const turnId = `turn-${index}` + await journal.appendItem( + { ...RUNNING_IDENTITY, turnId }, + { kind: 'turn', turnId, state: 'running', startedAt: 100 }, + { fence: 1 } + ) + } + now = 9_000 + const settle = () => + settleStaleStructuredAgentSessionState({ + journal, + sessionId: 'session-1', + fence: 2, + acquisitionGeneration: null, + deathEvidence: PROVEN + }) + const turns = () => + journal.snapshot().items.flatMap((item) => readAgentJournalTurn(item.body) ?? []) + const statusRows = () => + journal.snapshot().items.filter((item) => item.body.kind === 'status') + const append = journal.appendLifecycleBatch.bind(journal) + const secondBatchFails = vi + .spyOn(journal, 'appendLifecycleBatch') + .mockImplementationOnce(append) + .mockRejectedValueOnce(new Error('disk full')) + + await expect(settle()).rejects.toThrow('disk full') + secondBatchFails.mockRestore() + expect(turns().filter((turn) => turn.state === 'running')).toHaveLength(200) + expect(statusRows()).toHaveLength(1) + + await settle() + expect(turns().filter((turn) => turn.state !== 'interrupted')).toEqual([]) + expect(statusRows()).toHaveLength(1) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + }) + it('writes nothing when no turn is running and keys on the journal position without a generation', async () => { const idle = journalWith([ lifecycleItem('turn-1', 'completed', 1, { startedAt: 10, completedAt: 20 }) From e5cf5a0b89706e79daf963d62442fdcce0aa7c6f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:50:13 -0700 Subject: [PATCH 22/24] fix(native-chat): end a crashed turn at its owner's provider output, never at a later send A send accepted into a crashed chat before the proof of death wrote a submission row live, and the journal-wide last-live-row bound counted it, so the revised turn ended at the send and counted Orca's downtime. The bound is now the last row the owner's provider child wrote, per writer fence: submissions, dispatch rows and crash reconciliation are Orca's or the user's, and a newer owner's work says nothing about the dead one. --- .../agent-session-journal/journal-reducer.ts | 30 +++- .../agent-session-journal/journal-store.ts | 4 +- ...d-agent-session-stale-turn-verdict.test.ts | 141 +++++++++++++++--- ...ctured-agent-session-stale-turn-verdict.ts | 9 +- 4 files changed, 157 insertions(+), 27 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index bbe16467fc78..91aff8c5d462 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -38,8 +38,8 @@ export type JournalReducerState = { epoch: string lastSequence: number lastActivityAt: number - /** `lastActivityAt` over rows written live: crash reconciliation is not the provider working. */ - lastLiveActivityAt: number + /** Writer fence → the last row that owner's provider child wrote live: its last sign of work. */ + providerActivityAt: Map /** Lowest sequence still individually replayable; rows below it were compacted. */ oldestSequence: number highestFence: number @@ -62,7 +62,7 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou epoch, lastSequence: 0, lastActivityAt: 0, - lastLiveActivityAt: 0, + providerActivityAt: new Map(), oldestSequence: 1, highestFence: 0, items: new Map(), @@ -82,7 +82,10 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } state.lastActivityAt = Math.max(state.lastActivityAt, row.ts) - state.lastLiveActivityAt = Math.max(state.lastLiveActivityAt, row.recovered ? 0 : row.ts) + if (isLiveProviderOutput(row)) { + const last = state.providerActivityAt.get(row.fence) ?? 0 + state.providerActivityAt.set(row.fence, Math.max(last, row.ts)) + } if (row.kind === 'item') { if (journalItemRevisionIsStale(state, row.itemId, row.revision)) { return @@ -131,6 +134,25 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo applyJournalDispatchRow(state, row) } +/** Whether a row says the provider child was working. Exhaustive, so a new kind must be classified. */ +function isLiveProviderOutput(row: JournalRow): boolean { + // Crash reconciliation is Orca writing after the fact. + if (row.recovered) { + return false + } + switch (row.kind) { + case 'item': + case 'tombstone': + case 'lifecycle-batch': + return true + // Orca's and the user's rows: a send is accepted, and journaled, after the child died too. + case 'submission': + case 'dispatch': + case 'epoch': + return false + } +} + export function rememberAppliedSettlementId( state: JournalReducerState, settlementId: string diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 2639220ea2d3..5a891477ce8f 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -234,8 +234,8 @@ export class AgentSessionJournal { /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt - /** The same over rows written live, so crash reconciliation never reads as activity; 0 if none. */ - lastLiveActivityAt = (): number => this.state.lastLiveActivityAt + /** When the provider child of the owner at `fence` last wrote a row live; 0 if never. */ + lastProviderActivityAt = (fence: number): number => this.state.providerActivityAt.get(fence) ?? 0 /** Fence of the writer that created the item, while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 214512605360..daa5b908ef49 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -81,37 +81,42 @@ function promptItem(state: 'pending' | 'resolved', sequence: number): AgentJourn } describe('turn verdict from death evidence', () => { - const lastLiveAt = (at: number) => ({ lastLiveActivityAt: () => at }) + const providerActiveAt = (at: number) => ({ lastProviderActivityAt: () => at }) /** Judges a turn the fence-1 owner wrote, by evidence naming that owner. */ const verdictForTurn = ( evidence: AgentSessionDeathEvidence | null | undefined, - journal: ReturnType + journal: ReturnType ) => turnVerdictFromDeathEvidence(evidence && { ownerFence: 1, ...evidence }, journal, 1) it('ends a watched exit at the exit', () => { expect( - verdictForTurn({ kind: 'exit-observed', detail: 'exit', observedAt: 500 }, lastLiveAt(300)) + verdictForTurn( + { kind: 'exit-observed', detail: 'exit', observedAt: 500 }, + providerActiveAt(300) + ) ).toEqual({ state: 'interrupted', completedAt: 500 }) }) it.each(['pid-absent', 'identity-mismatch'] as const)( - 'ends a %s proof with no proof of life at the last row the journal saw live, not at the probe', + 'ends a %s proof with no proof of life at the last row the provider wrote, not at the probe', (kind) => { // Probed at 9000, long after the crash: the downtime is never counted as work. - expect(verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, lastLiveAt(300))).toEqual({ + expect( + verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, providerActiveAt(300)) + ).toEqual({ state: 'interrupted', completedAt: 300 }) // A live row stamped after the probe cannot outlast it, and no live row leaves only the probe. for (const lastLive of [9_500, 0]) { expect( - verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, lastLiveAt(lastLive)) + verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, providerActiveAt(lastLive)) ).toEqual({ state: 'interrupted', completedAt: 9_000 }) } } ) - it('ends a probe-proven death at the later of the last renewal and the last live row', () => { + it('ends a probe-proven death at the later of the last renewal and the last row the provider wrote', () => { const proof = (lastProvenAliveAt: number) => ({ kind: 'pid-absent' as const, detail: 'gone', @@ -119,21 +124,21 @@ describe('turn verdict from death evidence', () => { lastProvenAliveAt }) // A silent tool run: the renewal saw the child working long after its last row. - expect(verdictForTurn(proof(8_000), lastLiveAt(300))).toEqual({ + expect(verdictForTurn(proof(8_000), providerActiveAt(300))).toEqual({ state: 'interrupted', completedAt: 8_000 }) - expect(verdictForTurn(proof(8_000), lastLiveAt(0))).toEqual({ + expect(verdictForTurn(proof(8_000), providerActiveAt(0))).toEqual({ state: 'interrupted', completedAt: 8_000 }) // A chatty provider: its last row is the tighter bound. - expect(verdictForTurn(proof(200), lastLiveAt(300))).toEqual({ + expect(verdictForTurn(proof(200), providerActiveAt(300))).toEqual({ state: 'interrupted', completedAt: 300 }) // Neither bound outlasts the probe. - expect(verdictForTurn(proof(9_500), lastLiveAt(300))).toEqual({ + expect(verdictForTurn(proof(9_500), providerActiveAt(300))).toEqual({ state: 'interrupted', completedAt: 9_000 }) @@ -143,14 +148,14 @@ describe('turn verdict from death evidence', () => { expect( verdictForTurn( { kind: 'exit-observed', detail: 'exit', observedAt: 500, lastProvenAliveAt: 400 }, - lastLiveAt(300) + providerActiveAt(300) ) ).toEqual({ state: 'interrupted', completedAt: 500 }) }) it('leaves a release nothing proved unverifiable', () => { - expect(verdictForTurn(null, lastLiveAt(300))).toEqual({ state: 'unverifiable' }) - expect(verdictForTurn(undefined, lastLiveAt(300))).toEqual({ + expect(verdictForTurn(null, providerActiveAt(300))).toEqual({ state: 'unverifiable' }) + expect(verdictForTurn(undefined, providerActiveAt(300))).toEqual({ state: 'unverifiable' }) }) @@ -158,10 +163,10 @@ describe('turn verdict from death evidence', () => { // Only a watched exit was proof then; a probe's proof stays unverifiable. const legacy = { detail: 'gone', observedAt: 9_000, lastProvenAliveAt: 8_000 } expect( - turnVerdictFromDeathEvidence({ ...legacy, kind: 'exit-observed' }, lastLiveAt(300), 1) + turnVerdictFromDeathEvidence({ ...legacy, kind: 'exit-observed' }, providerActiveAt(300), 1) ).toEqual({ state: 'interrupted', completedAt: 9_000 }) expect( - turnVerdictFromDeathEvidence({ ...legacy, kind: 'pid-absent' }, lastLiveAt(300), 1) + turnVerdictFromDeathEvidence({ ...legacy, kind: 'pid-absent' }, providerActiveAt(300), 1) ).toEqual({ state: 'unverifiable' }) }) @@ -179,7 +184,7 @@ describe('turn verdict from death evidence', () => { [{ ...proof, kind: 'exit-observed' as const, ownerFence: 3 }, 1], [{ ...proof, ownerFence: 1 }, undefined] ] as const) { - expect(turnVerdictFromDeathEvidence(evidence, lastLiveAt(300), turnFence)).toEqual({ + expect(turnVerdictFromDeathEvidence(evidence, providerActiveAt(300), turnFence)).toEqual({ state: 'unverifiable' }) } @@ -417,7 +422,7 @@ describe('stale session state on a cold acquire', () => { } }) - it('ends a probe-proven turn at its last live row, which a revised item does not carry', async () => { + it('ends a probe-proven turn at the last row the provider wrote, which a revised item does not carry', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) const journals = createTrackedJournalOpener() let now = 100 @@ -548,6 +553,106 @@ describe('stale session state on a cold acquire', () => { } }) + describe('a crashed turn revised by a proof that lands after the relaunch', () => { + const CRASH_BOUND = { + kind: 'pid-absent' as const, + detail: 'gone', + observedAt: 3_606_000, + ownerFence: 1, + lastProvenAliveAt: 210 + } + + /** The fence-1 owner worked until 200 and died; the relaunch's open found no proof yet. */ + async function crashedThenOpened( + root: string, + journals: ReturnType + ) { + let now = 100 + const journal = await journals.open({ + identity: { + sessionId: 'session-1', + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: THREAD } + }, + journalDir: root, + now: () => now + }) + await journal.appendItem( + RUNNING_IDENTITY, + { kind: 'turn', turnId: 'turn-2', state: 'running', startedAt: 100 }, + { fence: 1 } + ) + now = 200 + await journal.appendItem( + { ...RUNNING_IDENTITY, ordinal: 1 }, + { kind: 'tool-call', name: 'shell', input: { command: 'pnpm test' }, state: 'running' }, + { fence: 1 } + ) + now = 3_600_000 + const settle = (fence: number, deathEvidence: AgentSessionDeathEvidence | null) => + settleStaleStructuredAgentSessionState({ + journal, + sessionId: 'session-1', + fence, + acquisitionGeneration: null, + deathEvidence + }) + await settle(1, null) + const turn = () => journal.snapshot().items.map((item) => readAgentJournalTurn(item.body))[0] + expect(turn()).toMatchObject({ state: 'unverifiable' }) + return { journal, settle, turn, at: (time: number) => (now = time) } + } + + it('does not end the turn at a send accepted before the proof', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) + const journals = createTrackedJournalOpener() + try { + const { journal, settle, turn, at } = await crashedThenOpened(root, journals) + at(3_605_000) + await journal.appendSubmission({ + clientMessageId: 'send-1', + payloadFingerprint: 'fingerprint-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'still there?' }] }, + fence: 1, + handoverRecorded: true + }) + await journal.resolveDispatch({ clientMessageId: 'send-1', state: 'pending', fence: 1 }) + at(3_606_000) + + await settle(1, CRASH_BOUND) + + expect(turn()).toMatchObject({ state: 'interrupted', completedAt: 210 }) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + }) + + it("does not end the turn at a newer owner's work", async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) + const journals = createTrackedJournalOpener() + try { + const { journal, settle, turn, at } = await crashedThenOpened(root, journals) + at(3_605_000) + await journal.appendItem( + { ...RUNNING_IDENTITY, turnId: 'turn-3' }, + { kind: 'turn', turnId: 'turn-3', state: 'running', startedAt: 3_605_000 }, + { fence: 2 } + ) + at(3_606_000) + + await settle(2, CRASH_BOUND) + + expect(turn()).toMatchObject({ state: 'interrupted', completedAt: 210 }) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } + }) + }) + it('revises an unverifiable turn only from a proof naming its own owner, and only once', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) const journals = createTrackedJournalOpener() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 8649b0083cb4..7fd39ecee2f3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -29,7 +29,7 @@ export const UNVERIFIABLE_TURN_VERDICT: StructuredAgentSessionTurnVerdict = { export function turnVerdictFromDeathEvidence( evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick, + journal: Pick, /** Fence of the owner that wrote the turn. */ turnFence: number | undefined ): StructuredAgentSessionTurnVerdict { @@ -50,7 +50,10 @@ export function turnVerdictFromDeathEvidence( } // A probe finds a dead child long after it died. The later of the last renewal and the last row // it wrote bounds its end, so the turn never counts the time Orca itself was down. - const lastSeen = Math.max(evidence.lastProvenAliveAt ?? 0, journal.lastLiveActivityAt()) + const lastSeen = Math.max( + evidence.lastProvenAliveAt ?? 0, + journal.lastProviderActivityAt(evidence.ownerFence) + ) return { state: 'interrupted', completedAt: lastSeen > 0 ? Math.min(lastSeen, evidence.observedAt) : evidence.observedAt @@ -76,7 +79,7 @@ export function runningTurnLifecycleRevisions( export function provenUnverifiableTurnRevisions( items: readonly AgentJournalRenderItem[], evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick + journal: Pick ): JournalLifecycleMutationInput[] { const ownerFence = evidence?.ownerFence if (ownerFence === undefined) { From ed7cce41c15db0e8b7cee456e220aaed79f32571 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:57:22 -0700 Subject: [PATCH 23/24] fix(native-chat): end a crashed turn at its last proof of life, never at a timeline row The end of a probe-proven death was the later of the last renewal and the last live timeline row. A send accepted into a crashed chat before the proof writes a row live, so the revised turn ended at the send and counted Orca's downtime. Rows cannot tell the agent's output from Orca's or the user's, so the end is now the last renewal alone, never after the probe, and never before the turn began. The journal's live-activity bound and the reopen's recovered marker, which existed only for it, are gone. --- .../agent-session-journal/journal-reducer.ts | 26 -- .../journal-store-restore.ts | 4 +- .../agent-session-journal/journal-store.ts | 3 - ...gent-session-dead-generation-settlement.ts | 2 +- ...d-agent-session-stale-turn-verdict.test.ts | 286 +++++------------- ...ctured-agent-session-stale-turn-verdict.ts | 28 +- ...t-session-wedged-profile-migration.test.ts | 22 +- 7 files changed, 99 insertions(+), 272 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 91aff8c5d462..d7e22575702d 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -38,8 +38,6 @@ export type JournalReducerState = { epoch: string lastSequence: number lastActivityAt: number - /** Writer fence → the last row that owner's provider child wrote live: its last sign of work. */ - providerActivityAt: Map /** Lowest sequence still individually replayable; rows below it were compacted. */ oldestSequence: number highestFence: number @@ -62,7 +60,6 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou epoch, lastSequence: 0, lastActivityAt: 0, - providerActivityAt: new Map(), oldestSequence: 1, highestFence: 0, items: new Map(), @@ -82,10 +79,6 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } state.lastActivityAt = Math.max(state.lastActivityAt, row.ts) - if (isLiveProviderOutput(row)) { - const last = state.providerActivityAt.get(row.fence) ?? 0 - state.providerActivityAt.set(row.fence, Math.max(last, row.ts)) - } if (row.kind === 'item') { if (journalItemRevisionIsStale(state, row.itemId, row.revision)) { return @@ -134,25 +127,6 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo applyJournalDispatchRow(state, row) } -/** Whether a row says the provider child was working. Exhaustive, so a new kind must be classified. */ -function isLiveProviderOutput(row: JournalRow): boolean { - // Crash reconciliation is Orca writing after the fact. - if (row.recovered) { - return false - } - switch (row.kind) { - case 'item': - case 'tombstone': - case 'lifecycle-batch': - return true - // Orca's and the user's rows: a send is accepted, and journaled, after the child died too. - case 'submission': - case 'dispatch': - case 'epoch': - return false - } -} - export function rememberAppliedSettlementId( state: JournalReducerState, settlementId: string diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index ffa63e436a83..fc69dd339d8b 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -39,9 +39,7 @@ export function restoreJournalStore( publishRepairEpoch: () => collaborators.epochController.start('unreconcilable_prefix', host.state().highestFence), adopt: host.adopt, - // What an open says about a host that is gone is crash reconciliation, never the provider working. - appendItem: (identity, body, fence) => - host.journal().appendItem(identity, body, { fence, recovered: true }), + appendItem: (identity, body, fence) => host.journal().appendItem(identity, body, { fence }), agent: host.identity.agent, highestFence: () => host.state().highestFence, malformedRows: host.malformedRows, diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 5a891477ce8f..3be3aebbc514 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -234,9 +234,6 @@ export class AgentSessionJournal { /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt - /** When the provider child of the owner at `fence` last wrote a row live; 0 if never. */ - lastProviderActivityAt = (fence: number): number => this.state.providerActivityAt.get(fence) ?? 0 - /** Fence of the writer that created the item, while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 2050981f68be..8ee3430c5c91 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -203,7 +203,7 @@ export async function settleStaleStructuredAgentSessionState(input: { const items = journal.snapshot().items // Each turn is judged by the evidence only if it names that turn's owner. const verdictFor = (item: AgentJournalRenderItem) => - turnVerdictFromDeathEvidence(input.deathEvidence, journal, journal.itemFence(item.itemId)) + turnVerdictFromDeathEvidence(input.deathEvidence, journal.itemFence(item.itemId)) // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index daa5b908ef49..fdf5087319f5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -6,10 +6,6 @@ import { agentJournalItemKey } from '../../../shared/agent-session-journal-item- import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' -import { - codexSubagentGroupBody, - codexSubagentGroupIdentity -} from '../../codex/codex-subagent-roster' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' @@ -81,93 +77,59 @@ function promptItem(state: 'pending' | 'resolved', sequence: number): AgentJourn } describe('turn verdict from death evidence', () => { - const providerActiveAt = (at: number) => ({ lastProviderActivityAt: () => at }) /** Judges a turn the fence-1 owner wrote, by evidence naming that owner. */ - const verdictForTurn = ( - evidence: AgentSessionDeathEvidence | null | undefined, - journal: ReturnType - ) => turnVerdictFromDeathEvidence(evidence && { ownerFence: 1, ...evidence }, journal, 1) + const verdictForTurn = (evidence: AgentSessionDeathEvidence | null | undefined) => + turnVerdictFromDeathEvidence(evidence && { ownerFence: 1, ...evidence }, 1) it('ends a watched exit at the exit', () => { - expect( - verdictForTurn( - { kind: 'exit-observed', detail: 'exit', observedAt: 500 }, - providerActiveAt(300) - ) - ).toEqual({ state: 'interrupted', completedAt: 500 }) + expect(verdictForTurn({ kind: 'exit-observed', detail: 'exit', observedAt: 500 })).toEqual({ + state: 'interrupted', + completedAt: 500 + }) }) it.each(['pid-absent', 'identity-mismatch'] as const)( - 'ends a %s proof with no proof of life at the last row the provider wrote, not at the probe', + 'ends a %s proof at the last proof of life, never after the probe', (kind) => { - // Probed at 9000, long after the crash: the downtime is never counted as work. - expect( - verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, providerActiveAt(300)) - ).toEqual({ - state: 'interrupted', - completedAt: 300 + const proof = (lastProvenAliveAt?: number) => ({ + kind, + detail: 'gone', + observedAt: 9_000, + ...(lastProvenAliveAt === undefined ? {} : { lastProvenAliveAt }) }) - // A live row stamped after the probe cannot outlast it, and no live row leaves only the probe. - for (const lastLive of [9_500, 0]) { - expect( - verdictForTurn({ kind, detail: 'gone', observedAt: 9_000 }, providerActiveAt(lastLive)) - ).toEqual({ state: 'interrupted', completedAt: 9_000 }) - } + // Probed at 9000, long after the crash: the downtime is never counted as work. + expect(verdictForTurn(proof(8_000))).toEqual({ state: 'interrupted', completedAt: 8_000 }) + expect(verdictForTurn(proof(9_500))).toEqual({ state: 'interrupted', completedAt: 9_000 }) + // A proof that recorded no proof of life has only the probe. + expect(verdictForTurn(proof())).toEqual({ state: 'interrupted', completedAt: 9_000 }) } ) - it('ends a probe-proven death at the later of the last renewal and the last row the provider wrote', () => { - const proof = (lastProvenAliveAt: number) => ({ - kind: 'pid-absent' as const, - detail: 'gone', - observedAt: 9_000, - lastProvenAliveAt - }) - // A silent tool run: the renewal saw the child working long after its last row. - expect(verdictForTurn(proof(8_000), providerActiveAt(300))).toEqual({ - state: 'interrupted', - completedAt: 8_000 - }) - expect(verdictForTurn(proof(8_000), providerActiveAt(0))).toEqual({ - state: 'interrupted', - completedAt: 8_000 - }) - // A chatty provider: its last row is the tighter bound. - expect(verdictForTurn(proof(200), providerActiveAt(300))).toEqual({ - state: 'interrupted', - completedAt: 300 - }) - // Neither bound outlasts the probe. - expect(verdictForTurn(proof(9_500), providerActiveAt(300))).toEqual({ - state: 'interrupted', - completedAt: 9_000 - }) - }) - it('ends a watched exit at the exit even when a renewal is recorded', () => { expect( - verdictForTurn( - { kind: 'exit-observed', detail: 'exit', observedAt: 500, lastProvenAliveAt: 400 }, - providerActiveAt(300) - ) + verdictForTurn({ + kind: 'exit-observed', + detail: 'exit', + observedAt: 500, + lastProvenAliveAt: 400 + }) ).toEqual({ state: 'interrupted', completedAt: 500 }) }) it('leaves a release nothing proved unverifiable', () => { - expect(verdictForTurn(null, providerActiveAt(300))).toEqual({ state: 'unverifiable' }) - expect(verdictForTurn(undefined, providerActiveAt(300))).toEqual({ - state: 'unverifiable' - }) + expect(verdictForTurn(null)).toEqual({ state: 'unverifiable' }) + expect(verdictForTurn(undefined)).toEqual({ state: 'unverifiable' }) }) it('keeps the rule an older build applied to evidence that names no owner', () => { // Only a watched exit was proof then; a probe's proof stays unverifiable. const legacy = { detail: 'gone', observedAt: 9_000, lastProvenAliveAt: 8_000 } - expect( - turnVerdictFromDeathEvidence({ ...legacy, kind: 'exit-observed' }, providerActiveAt(300), 1) - ).toEqual({ state: 'interrupted', completedAt: 9_000 }) - expect( - turnVerdictFromDeathEvidence({ ...legacy, kind: 'pid-absent' }, providerActiveAt(300), 1) - ).toEqual({ state: 'unverifiable' }) + expect(turnVerdictFromDeathEvidence({ ...legacy, kind: 'exit-observed' }, 1)).toEqual({ + state: 'interrupted', + completedAt: 9_000 + }) + expect(turnVerdictFromDeathEvidence({ ...legacy, kind: 'pid-absent' }, 1)).toEqual({ + state: 'unverifiable' + }) }) it('gives a turn no end from evidence about any other owner', () => { @@ -184,9 +146,7 @@ describe('turn verdict from death evidence', () => { [{ ...proof, kind: 'exit-observed' as const, ownerFence: 3 }, 1], [{ ...proof, ownerFence: 1 }, undefined] ] as const) { - expect(turnVerdictFromDeathEvidence(evidence, providerActiveAt(300), turnFence)).toEqual({ - state: 'unverifiable' - }) + expect(turnVerdictFromDeathEvidence(evidence, turnFence)).toEqual({ state: 'unverifiable' }) } }) }) @@ -264,6 +224,13 @@ describe('running turn lifecycle revisions', () => { }) }) + it('never ends a turn before it began, when the last proof of life predates it', () => { + const item = lifecycleItem('turn-2', 'running', 2, { startedAt: 500 }) + expect( + runningTurnLifecycleRevisions([item], { state: 'interrupted', completedAt: 300 }) + ).toMatchObject([{ body: { kind: 'turn', state: 'interrupted', completedAt: 500 } }]) + }) + it('revises a legacy status-form running row from an older host into a typed turn', () => { expect( runningTurnLifecycleRevisions([legacyLifecycleItem('turn-2', 30)], { state: 'unverifiable' }) @@ -282,12 +249,13 @@ describe('running turn lifecycle revisions', () => { }) }) -/** A probe that found the fence-1 owner gone at 9000. */ +/** A probe that found the fence-1 owner gone at 9000, last proven alive at 100. */ const PROVEN: AgentSessionDeathEvidence = { kind: 'pid-absent', detail: 'recorded pid absent on host', observedAt: 9_000, - ownerFence: 1 + ownerFence: 1, + lastProvenAliveAt: 100 } describe('stale session state on a cold acquire', () => { @@ -422,7 +390,7 @@ describe('stale session state on a cold acquire', () => { } }) - it('ends a probe-proven turn at the last row the provider wrote, which a revised item does not carry', async () => { + it('ends a probe-proven turn at its last proof of life, not at a row written after it', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) const journals = createTrackedJournalOpener() let now = 100 @@ -447,27 +415,13 @@ describe('stale session state on a cold acquire', () => { ) now = 200 await journal.appendItem(command, { ...shell, state: 'running' }, { fence: 1 }) - // Output streamed until 700: a revision, so the item still reads as first seen at 200. + // Rows can outlast the last renewal; only the renewal is proof of life. now = 700 await journal.appendItem( command, { ...shell, input: { command: 'pnpm test', streamed: 'ok' }, state: 'running' }, { fence: 1 } ) - // Crash reconciliation is Orca writing, not the provider working. - now = 900 - await journal.appendLifecycleBatch({ - settlementId: 'earlier-recovery', - fence: 1, - recovered: true, - mutations: [ - { - kind: 'item', - identity: { provider: 'orca', clientMessageId: 'earlier-recovery' }, - body: { kind: 'status', text: 'recovered' } - } - ] - }) now = 9_000 await settleStaleStructuredAgentSessionState({ @@ -479,20 +433,20 @@ describe('stale session state on a cold acquire', () => { kind: 'pid-absent', detail: 'recorded pid absent on host', observedAt: 9_000, - ownerFence: 1 + ownerFence: 1, + lastProvenAliveAt: 650 } }) const items = journal.snapshot().items expect(items.map((item) => readAgentJournalTurn(item.body)).find(Boolean)).toMatchObject({ state: 'interrupted', - completedAt: 700 + completedAt: 650 }) // The probe's detail is Orca's, so the row carries none. expect( items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) ).toEqual([ - 'recovered', 'The agent stopped while this response was in progress. You can continue in this conversation.' ]) } finally { @@ -501,73 +455,11 @@ describe('stale session state on a cold acquire', () => { } }) - it('does not count the reopen settling a roster the crashed host left working as activity', async () => { + it('ends a crashed turn at its last proof of life, not at a send accepted before the proof', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) const journals = createTrackedJournalOpener() let now = 100 - const identity = { - sessionId: 'session-1', - workspaceId: 'workspace-1', - hostId: 'local', - agent: 'codex' as const, - providerHandle: { kind: 'codex' as const, threadId: THREAD } - } try { - const live = await journals.open({ identity, journalDir: root, now: () => now }) - await live.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 0 }, - { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: 100 }, - { fence: 1 } - ) - now = 200 - const groupId = `${THREAD}:turn-1` - await live.appendItem( - codexSubagentGroupIdentity(groupId), - codexSubagentGroupBody(groupId, [ - { id: 'a', label: 'explore', state: 'working', startedAt: 200 } - ]), - { fence: 1 } - ) - await live.close() - // Relaunched long after the crash: the open retires the roster, then the probe proves death. - now = 9_000 - const reopened = await journals.open({ identity, journalDir: root, now: () => now }) - - await settleStaleStructuredAgentSessionState({ - journal: reopened, - sessionId: 'session-1', - fence: 2, - acquisitionGeneration: 'generation-2', - deathEvidence: { kind: 'pid-absent', detail: 'gone', observedAt: 8_500, ownerFence: 1 } - }) - - expect( - reopened - .snapshot() - .items.map((item) => readAgentJournalTurn(item.body)) - .find(Boolean) - ).toMatchObject({ state: 'interrupted', completedAt: 200 }) - } finally { - await journals.closeAll() - await rm(root, { recursive: true, force: true }) - } - }) - - describe('a crashed turn revised by a proof that lands after the relaunch', () => { - const CRASH_BOUND = { - kind: 'pid-absent' as const, - detail: 'gone', - observedAt: 3_606_000, - ownerFence: 1, - lastProvenAliveAt: 210 - } - - /** The fence-1 owner worked until 200 and died; the relaunch's open found no proof yet. */ - async function crashedThenOpened( - root: string, - journals: ReturnType - ) { - let now = 100 const journal = await journals.open({ identity: { sessionId: 'session-1', @@ -590,67 +482,43 @@ describe('stale session state on a cold acquire', () => { { kind: 'tool-call', name: 'shell', input: { command: 'pnpm test' }, state: 'running' }, { fence: 1 } ) - now = 3_600_000 - const settle = (fence: number, deathEvidence: AgentSessionDeathEvidence | null) => + const settle = (deathEvidence: AgentSessionDeathEvidence | null) => settleStaleStructuredAgentSessionState({ journal, sessionId: 'session-1', - fence, + fence: 1, acquisitionGeneration: null, deathEvidence }) - await settle(1, null) + // An hour later the relaunch opens the chat before anything proved the owner gone. + now = 3_600_000 + await settle(null) const turn = () => journal.snapshot().items.map((item) => readAgentJournalTurn(item.body))[0] expect(turn()).toMatchObject({ state: 'unverifiable' }) - return { journal, settle, turn, at: (time: number) => (now = time) } - } - - it('does not end the turn at a send accepted before the proof', async () => { - const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) - const journals = createTrackedJournalOpener() - try { - const { journal, settle, turn, at } = await crashedThenOpened(root, journals) - at(3_605_000) - await journal.appendSubmission({ - clientMessageId: 'send-1', - payloadFingerprint: 'fingerprint-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'still there?' }] }, - fence: 1, - handoverRecorded: true - }) - await journal.resolveDispatch({ clientMessageId: 'send-1', state: 'pending', fence: 1 }) - at(3_606_000) - - await settle(1, CRASH_BOUND) - - expect(turn()).toMatchObject({ state: 'interrupted', completedAt: 210 }) - } finally { - await journals.closeAll() - await rm(root, { recursive: true, force: true }) - } - }) - - it("does not end the turn at a newer owner's work", async () => { - const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) - const journals = createTrackedJournalOpener() - try { - const { journal, settle, turn, at } = await crashedThenOpened(root, journals) - at(3_605_000) - await journal.appendItem( - { ...RUNNING_IDENTITY, turnId: 'turn-3' }, - { kind: 'turn', turnId: 'turn-3', state: 'running', startedAt: 3_605_000 }, - { fence: 2 } - ) - at(3_606_000) - - await settle(2, CRASH_BOUND) + now = 3_605_000 + await journal.appendSubmission({ + clientMessageId: 'send-1', + payloadFingerprint: 'fingerprint-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'still there?' }] }, + fence: 1, + handoverRecorded: true + }) + await journal.resolveDispatch({ clientMessageId: 'send-1', state: 'pending', fence: 1 }) + now = 3_606_000 + + await settle({ + kind: 'pid-absent', + detail: 'gone', + observedAt: 3_606_000, + ownerFence: 1, + lastProvenAliveAt: 210 + }) - expect(turn()).toMatchObject({ state: 'interrupted', completedAt: 210 }) - } finally { - await journals.closeAll() - await rm(root, { recursive: true, force: true }) - } - }) + expect(turn()).toMatchObject({ state: 'interrupted', completedAt: 210 }) + } finally { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) + } }) it('revises an unverifiable turn only from a proof naming its own owner, and only once', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 7fd39ecee2f3..b23018aeba83 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -29,7 +29,6 @@ export const UNVERIFIABLE_TURN_VERDICT: StructuredAgentSessionTurnVerdict = { export function turnVerdictFromDeathEvidence( evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick, /** Fence of the owner that wrote the turn. */ turnFence: number | undefined ): StructuredAgentSessionTurnVerdict { @@ -48,15 +47,11 @@ export function turnVerdictFromDeathEvidence( if (evidence.kind === 'exit-observed') { return { state: 'interrupted', completedAt: evidence.observedAt } } - // A probe finds a dead child long after it died. The later of the last renewal and the last row - // it wrote bounds its end, so the turn never counts the time Orca itself was down. - const lastSeen = Math.max( - evidence.lastProvenAliveAt ?? 0, - journal.lastProviderActivityAt(evidence.ownerFence) - ) + // A probe finds a dead child long after it died; its last renewal bounds the end, so the turn never + // counts the time Orca was down. Timeline rows don't: a send can land there after the death. return { state: 'interrupted', - completedAt: lastSeen > 0 ? Math.min(lastSeen, evidence.observedAt) : evidence.observedAt + completedAt: Math.min(evidence.lastProvenAliveAt ?? evidence.observedAt, evidence.observedAt) } } @@ -79,7 +74,7 @@ export function runningTurnLifecycleRevisions( export function provenUnverifiableTurnRevisions( items: readonly AgentJournalRenderItem[], evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick + journal: Pick ): JournalLifecycleMutationInput[] { const ownerFence = evidence?.ownerFence if (ownerFence === undefined) { @@ -88,11 +83,7 @@ export function provenUnverifiableTurnRevisions( return items.flatMap((item) => { const turn = readAgentJournalTurn(item.body) return turn?.state === 'unverifiable' && journal.itemFence(item.itemId) === ownerFence - ? turnLifecycleRevision( - item, - turn, - turnVerdictFromDeathEvidence(evidence, journal, ownerFence) - ) + ? turnLifecycleRevision(item, turn, turnVerdictFromDeathEvidence(evidence, ownerFence)) : [] }) } @@ -121,7 +112,10 @@ function settledLifecycle( durationMs: _durationMs, ...kept } = lifecycle - return verdict.state === 'interrupted' - ? { ...kept, state: verdict.state, completedAt: verdict.completedAt } - : { ...kept, state: verdict.state } + if (verdict.state !== 'interrupted') { + return { ...kept, state: verdict.state } + } + // A renewal can predate the turn, which started with its owner alive; it never ends before that. + const began = Math.max(lifecycle.requestedAt ?? 0, lifecycle.startedAt ?? 0) + return { ...kept, state: verdict.state, completedAt: Math.max(verdict.completedAt, began) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index c49f41bef743..a492b0b65ce1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -191,8 +191,8 @@ function isAcquirable(lease: NonNullable>['le ) } -/** When the seeded turn's row was written: the last time the dead owner was seen working. */ -const SEEDED_TURN_SEEN_AT = NOW - 4_000 +/** After the owner's last renewal, so a turn it proves dead ends at its start, never before. */ +const SEEDED_TURN_STARTED_AT = NOW - 5_000 async function seedRunningTurn(provider: 'codex' | 'claude' = 'codex'): Promise { const journal = await openAgentSessionJournal({ @@ -206,17 +206,13 @@ async function seedRunningTurn(provider: 'codex' | 'claude' = 'codex'): Promise< ? { kind: 'codex', threadId: THREAD } : { kind: 'claude', sessionId: 'provider-session-alpha-1', leafUuid: null } }, - journalDir: journalDirectoryFor(root, { - workspaceId: LOCATION.workspaceId, - sessionId: SESSION - }), - now: () => SEEDED_TURN_SEEN_AT + journalDir: journalDirectoryFor(root, { workspaceId: LOCATION.workspaceId, sessionId: SESSION }) }) await journal.appendItem( provider === 'codex' ? { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 0 } : { provider: 'claude', sessionId: 'provider-session-alpha-1', uuid: 'uuid-running' }, - { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: NOW - 5_000 }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: SEEDED_TURN_STARTED_AT }, { fence: 13 } ) await journal.close() @@ -313,7 +309,7 @@ describe('already-wedged profiles become usable on load', () => { 'a quit that left the owner for a probe to prove gone', wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }), null, - { state: 'interrupted', completedAt: SEEDED_TURN_SEEN_AT } + { state: 'interrupted', completedAt: SEEDED_TURN_STARTED_AT } ], [ 'a quit that left an owner on a host this one cannot probe', @@ -393,12 +389,12 @@ describe('already-wedged profiles become usable on load', () => { expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) - // A pid probe proved the owner gone, so the turn was cut short when it was last seen working. + // A pid probe proved the owner gone, so the turn was cut short when it was last proven alive. expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', state: 'interrupted', startedAt: NOW - 5_000, - completedAt: SEEDED_TURN_SEEN_AT, + completedAt: SEEDED_TURN_STARTED_AT, recovered: true }) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -487,12 +483,12 @@ describe('already-wedged profiles become usable on load', () => { expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' }) - // A pid probe proved the owner gone, so the turn was cut short when it was last seen working. + // A pid probe proved the owner gone, so the turn was cut short when it was last proven alive. expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', state: 'interrupted', startedAt: NOW - 5_000, - completedAt: SEEDED_TURN_SEEN_AT, + completedAt: SEEDED_TURN_STARTED_AT, recovered: true }) } From 0b0beb57d666d3b84961cff9311a2d9a045cc9bb Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:31:04 -0700 Subject: [PATCH 24/24] test(native-chat): drop a stale reference to the removed live-activity bound --- .../structured-agent-session-stale-turn-verdict.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index fdf5087319f5..2c7a91e76378 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -261,7 +261,7 @@ const PROVEN: AgentSessionDeathEvidence = { describe('stale session state on a cold acquire', () => { function journalWith(items: AgentJournalRenderItem[]) { const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 9 })) - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the settle reads only these journal members, and a release that proved nothing never reads the live bound. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the settle reads only these journal members. const journal = { snapshot: () => ({ items }), itemFence: () => 1,