From c44126deb29b07ed257f7e04a098527da898d2ec Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:17:51 -0700 Subject: [PATCH 01/13] fix: preserve streamed turn duration after host restart --- .../journal-provider-activity.ts | 38 +++ .../agent-session-journal/journal-reducer.ts | 6 + .../agent-session-journal/journal-store.ts | 4 + ...gent-session-dead-generation-settlement.ts | 4 +- ...nt-session-restart-stream-duration.test.ts | 260 ++++++++++++++++++ ...d-agent-session-stale-turn-verdict.test.ts | 13 +- ...ctured-agent-session-stale-turn-verdict.ts | 21 +- ...ured-agent-session-unexpected-exit.test.ts | 1 + 8 files changed, 333 insertions(+), 14 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-provider-activity.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-provider-activity.ts b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts new file mode 100644 index 000000000000..7c778210e31a --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts @@ -0,0 +1,38 @@ +import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentJournalItemBody } from '../../../shared/agent-session-journal-types' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import type { JournalReducerState } from './journal-reducer' +import type { JournalRow } from './journal-row-schema' + +/** Provider revisions prove life after a renewal; client actions and recovery writes do not. */ +export function observeJournalProviderActivity( + state: Pick, + row: JournalRow, + itemId: string, + body: AgentJournalItemBody +): void { + if (row.recovered || !isProviderActivity(body)) { + return + } + const identity = parseAgentJournalItemKey(itemId) + if (!identity || identity.provider === 'orca') { + return + } + state.providerActivityAt.set( + row.fence, + Math.max(state.providerActivityAt.get(row.fence) ?? 0, row.ts) + ) +} + +function isProviderActivity(body: AgentJournalItemBody): boolean { + if (body.kind === 'message') { + return body.role === 'assistant' + } + if (body.kind === 'tool-call') { + return body.endedAs === undefined + } + if (body.kind === 'approval' || body.kind === 'question') { + return body.resolution.state === 'pending' + } + return readAgentJournalTurn(body)?.state === 'running' +} diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 264933b7fb33..d32ce339e8a2 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -26,6 +26,7 @@ import { agentSessionSendBodyFingerprint } from '../../../shared/structured-agen import { JournalDerivedTurnScope } from './journal-derived-turn-scope' import { removeJournalItem, statedOrDerivedTurnScope, upsertJournalItem } from './journal-item-fold' import { journalItemRevisionIsStale } from './journal-item-revision' +import { observeJournalProviderActivity } from './journal-provider-activity' import { isJournalStopOrResumeRow, type JournalRow } from './journal-row-schema' import { acceptSubmissionFromProviderItem, applyJournalSubmission } from './journal-submission-fold' import { applyJournalDispatchRow } from './journal-dispatch-reducer' @@ -50,6 +51,8 @@ export type JournalReducerState = { items: Map /** Fence of the writer that created each item: the generation a running turn belongs to. */ itemFences: Map + /** Latest provider observation per owner, rebuilt from rows including streaming revisions. */ + providerActivityAt: Map /** Revision of a removed item, so a late lower revision cannot resurrect it. */ tombstones: Map submissions: Map @@ -77,6 +80,7 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou highestFence: 0, items: new Map(), itemFences: new Map(), + providerActivityAt: new Map(), tombstones: new Map(), submissions: new Map(), receipts: new Map(), @@ -100,6 +104,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } const itemId = resolveJournalItemId(state, row.itemId, row.body) + observeJournalProviderActivity(state, row, row.itemId, row.body) acceptSubmissionFromProviderItem(state, row.itemId, itemId, row) upsertJournalItem( state, @@ -128,6 +133,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo continue } const { revision, body } = mutation + observeJournalProviderActivity(state, row, mutation.itemId, body) const itemId = resolveJournalItemId(state, mutation.itemId, body) acceptSubmissionFromProviderItem(state, mutation.itemId, itemId, row) const producer = journalBatchMutationProducer(row, mutation) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 06bdfba25fea..13da0881670e 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -240,6 +240,10 @@ export class AgentSessionJournal { /** Fence of the writer that created the item, while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) + /** Provider output at this fence, excluding client actions and recovery bookkeeping. */ + lastProviderActivityAt = (fence: number): number | undefined => + this.state.providerActivityAt.get(fence) + submissions = (): AgentJournalSubmission[] => [...this.state.submissions.values()] submission = (clientMessageId: string) => this.state.submissions.get(clientMessageId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index f3c8c5df9fea..2c3d1849cb55 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -37,7 +37,7 @@ import { provenUnverifiableTurnRevisions, provenUnverifiedToolCallRevisions, runningTurnLifecycleRevisions, - stopFoundTurnLiveAt, + lastProvenTurnLiveAt, turnVerdictFromDeathEvidence, watchedExitRevisions, type StructuredAgentSessionTurnVerdict, @@ -197,7 +197,7 @@ export async function settleStaleStructuredAgentSessionState(input: { turnVerdictFromDeathEvidence( input.deathEvidence, journal.itemFence(item.itemId), - stopFoundTurnLiveAt(journal, item) + lastProvenTurnLiveAt(journal, item) ) // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts new file mode 100644 index 000000000000..8b79e0e155ff --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts @@ -0,0 +1,260 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + completedStructuredAgentTurnSeconds, + selectStructuredAgentTurnTimings +} from '../../../shared/structured-agent-session-turn-timing' +import { withNativeChatCutTurnNotices } from '../../../shared/native-chat-cut-turn-notice' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_LOCATION, + HOST_TEST_NOW, + hostTestAttachParams +} from './structured-agent-session-host-test-data' +import { + createRestTestRig, + REST_TEST_CALLER, + restTestSend, + REST_TEST_SESSION as SESSION, + REST_TEST_THREAD as THREAD, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 +const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 +const TURN = { provider: 'codex' as const, threadId: THREAD, turnId: 'stream-turn', ordinal: 0 } +const REPLY = { ...TURN, ordinal: 1 } +const TURN_SCOPE = { kind: 'turn' as const, turnItemId: agentJournalItemKey(TURN) } + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await rig.dispose() + closeTestJournalHostDatabases() +}) + +async function streamBeforeCrash( + workspaceKind: 'folder' | 'git-worktree' = 'git-worktree' +): Promise { + const attached = await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(null, { location: { ...HOST_TEST_LOCATION, workspaceKind } }) + ) + if (!attached.ok) { + throw new Error(`attach refused: ${attached.refusal.code}`) + } + const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('long stream', attached.fence)) + if (!sent.ok) { + throw new Error(`send refused: ${sent.refusal.code}`) + } + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) + const events = rig.adapter.acquire.mock.calls[0]?.[0].events + if (!events) { + throw new Error('the provider has no event sink') + } + events.appendItem( + TURN, + { kind: 'turn', turnId: TURN.turnId, state: 'running', startedAt: HOST_TEST_NOW }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE, observedAt: HOST_TEST_NOW } + ) + for (let chunk = 1; chunk <= 11; chunk += 1) { + events.appendItem( + REPLY, + { + kind: 'message', + role: 'assistant', + state: 'running', + blocks: [{ type: 'text', text: `chunks 1-${chunk}` }] + }, + { turnScope: TURN_SCOPE, observedAt: HOST_TEST_NOW + chunk * 500 } + ) + await rig.host.flushStreamedEvents(SESSION) + } + expect(rig.store.getRecord(SESSION)?.lease.lastRenewedAt).toBe(HOST_TEST_NOW) +} + +/** Drops the old host without an exit callback or graceful turn settlement. */ +async function restartAfterCrash(): Promise { + const old = rig.host + old.stopDelivery() + const { lifetime, runtimeState, sessions } = old.collaboratorsForTests() + lifetime.idleSweep.dispose() + await runtimeState.stopLeaseRenewal() + runtimeState.currentEventSink(SESSION)?.close() + await sessions.get(SESSION)?.journal.close() + closeTestJournalHostDatabases() + rig.store = await openTestAgentSessionRecordStore(rig.root) + rig.host = new StructuredAgentSessionHost({ + ...old.deps, + store: rig.store, + journalDatabase: openTestJournalHostDatabase(rig.root), + now: () => RESTARTED_AT + }) +} + +async function expectSettledStream(): Promise { + const snapshot = await rig.host.journalSnapshot(SESSION) + expect( + snapshot.items.find((item) => item.itemId === agentJournalItemKey(REPLY))?.body + ).toMatchObject({ + kind: 'message', + blocks: [{ type: 'text', text: 'chunks 1-11' }] + }) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) + const [timing] = selectStructuredAgentTurnTimings(snapshot.items).values() + expect(completedStructuredAgentTurnSeconds(timing)).toBe(5) + expect( + withNativeChatCutTurnNotices(snapshot.items, { agentName: 'Codex' }).filter( + (item) => item.body.kind === 'status' + ) + ).toHaveLength(1) + expect(rig.adapter.acquire).toHaveBeenCalledOnce() +} + +describe('a provider stream cut short before its next lease renewal', () => { + it.each(['git-worktree', 'folder'] as const)( + 'settles and replays the duration exactly once in a %s workspace', + async (workspaceKind) => { + await streamBeforeCrash(workspaceKind) + const cursor = (await rig.host.journalSnapshot(SESSION)).cursor + await restartAfterCrash() + await rig.host.restoreReadableSessions() + + await expectSettledStream() + const events: AgentSessionSubscribeEvent[] = [] + const unsubscribe = await rig.host.subscribe({ + id: 'reconnecting-client', + sessionId: SESSION, + cursor, + emit: (event) => events.push(event) + }) + const replayedTurns = events.flatMap((event) => { + const items = + event.type === 'snapshot' + ? event.page.items + : event.type === 'batch' + ? event.batch.items + : [] + return items.flatMap((item) => readAgentJournalTurn(item.body) ?? []) + }) + expect(replayedTurns).toEqual([ + expect.objectContaining({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) + ]) + unsubscribe() + const settled = await rig.host.journalSnapshot(SESSION) + await rig.host.restoreReadableSessions() + expect(await rig.host.journalSnapshot(SESSION)).toEqual(settled) + + await restartAfterCrash() + await rig.host.restoreReadableSessions() + await expectSettledStream() + // A reopen can mark its queue, but never revises the settled turn again. + expect((await rig.host.journalSnapshot(SESSION)).items).toEqual(settled.items) + } + ) + + it('retains the output bound when a client opens the journal before owner reconciliation', async () => { + await streamBeforeCrash() + await restartAfterCrash() + const beforeProof = await rig.host.journalSnapshot(SESSION) + expect( + beforeProof.items.some((item) => readAgentJournalTurn(item.body)?.state === 'unverifiable') + ).toBe(true) + + await rig.host.reconcileRestartLeases() + await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) + + await expectSettledStream() + }) + + it('also preserves the interrupted turn and duration after a local desktop quit', async () => { + await streamBeforeCrash() + rig.clock.now = LAST_CHUNK_AT + await rig.host.flushAllStreamedEvents({ trigger: 'quit' }) + await restartAfterCrash() + await rig.host.restoreReadableSessions() + + await expectSettledStream() + expect(rig.adapter.closeSession).toHaveBeenCalledOnce() + }) + + it('leaves the end unverifiable when the execution host cannot prove owner death', async () => { + await streamBeforeCrash() + await restartAfterCrash() + rig.host.deps.probeOwner = async () => ({ + outcome: 'indeterminate', + reason: 'probe unavailable' + }) + await rig.host.restoreReadableSessions() + + const snapshot = await rig.host.journalSnapshot(SESSION) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn?.state).toBe('unverifiable') + expect(turn?.completedAt).toBeUndefined() + expect(rig.adapter.acquire).toHaveBeenCalledOnce() + }) + + it('ignores later client actions, recovery revisions, and another owner generation', async () => { + await streamBeforeCrash() + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal + if (!journal) { + throw new Error('the streaming journal is not open') + } + const later = RESTARTED_AT - 1_000 + await journal.appendItem( + { provider: 'orca', clientMessageId: 'late-client-send' }, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'next task' }] }, + { fence: 1, turnScope: TURN_SCOPE, observedAt: later } + ) + await journal.appendItem( + { ...REPLY, ordinal: 2 }, + { + kind: 'approval', + title: 'Run?', + detail: null, + options: [], + resolution: { + state: 'resolved', + selectedOptionId: 'allow', + resolvedBy: 'client', + resolvedAt: later + } + }, + { fence: 1, turnScope: TURN_SCOPE, observedAt: later } + ) + const originalReply = journal.itemBody(agentJournalItemKey(REPLY)) + if (!originalReply) { + throw new Error('the streamed reply is missing') + } + vi.spyOn(Date, 'now').mockReturnValue(later) + await journal.appendLifecycleBatch({ + settlementId: 'earlier-read-recovery', + fence: 1, + recovered: true, + mutations: [{ kind: 'item', identity: REPLY, body: originalReply, turnScope: TURN_SCOPE }] + }) + await journal.appendItem( + { ...REPLY, ordinal: 3 }, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'another owner' }] }, + { fence: 2, turnScope: TURN_SCOPE, observedAt: later } + ) + await restartAfterCrash() + await rig.host.restoreReadableSessions() + + await expectSettledStream() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 90e5e84cf861..131c01fab656 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -277,6 +277,7 @@ describe('stale session state on a cold acquire', () => { const journal = { snapshot: () => ({ items }), itemFence: () => 1, + lastProviderActivityAt: () => undefined, stopMarks: { latest: () => null }, cursor: () => ({ epoch: 'epoch-1', sequence: 8 }), appendLifecycleBatch @@ -412,7 +413,7 @@ describe('stale session state on a cold acquire', () => { } }) - it('ends a probe-proven turn at its last proof of life, not at a row written after it', async () => { + it('uses provider output after a renewal, excluding a later client message', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-stale-session-')) const journals = createTrackedJournalOpener() let now = 100 @@ -438,13 +439,19 @@ describe('stale session state on a cold acquire', () => { ) now = 200 await journal.appendItem(command, { ...shell, state: 'running' }, { fence: 1, turnScope }) - // Rows can outlast the last renewal; only the renewal is proof of life. + // Streaming revisions prove the provider worked after its last lease renewal. now = 700 await journal.appendItem( command, { ...shell, input: { command: 'pnpm test', streamed: 'ok' }, state: 'running' }, { fence: 1, turnScope } ) + now = 800 + await journal.appendItem( + { provider: 'orca', clientMessageId: 'late-send' }, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'next task' }] }, + { fence: 1, turnScope } + ) now = 9_000 await settleStaleStructuredAgentSessionState({ @@ -464,7 +471,7 @@ describe('stale session state on a cold acquire', () => { const items = journal.snapshot().items expect(items.map((item) => readAgentJournalTurn(item.body)).find(Boolean)).toMatchObject({ state: 'interrupted', - completedAt: 650 + completedAt: 700 }) // The probe's detail is Orca's, so the row carries none. expect( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 2491eb2f2be2..7f6868cca5b9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -36,7 +36,7 @@ export function turnVerdictFromDeathEvidence( evidence: AgentSessionDeathEvidence | null | undefined, /** Fence of the owner that wrote the turn. */ turnFence: number | undefined, - /** When a Stop event found the turn running (`stopFoundTurnLiveAt`): a later proof of life. */ + /** A provider observation or Stop that found the turn running: a later proof of life. */ liveAt?: number ): StructuredAgentSessionTurnVerdict { if (!evidence) { @@ -55,20 +55,23 @@ export function turnVerdictFromDeathEvidence( return { state: 'interrupted', completedAt: evidence.observedAt } } // A probe finds a dead child long after it died; its last renewal bounds the end, so the turn never - // counts the time Orca was down. Timeline rows don't: a send can land there after the death. A - // Stop that found the turn running is a later renewal, so the end reads after that Stop. + // counts the time Orca was down. Provider output and a Stop that found the turn running can + // prove life after that renewal; a client's later send or a recovery write cannot. const lastAlive = Math.max(evidence.lastProvenAliveAt ?? evidence.observedAt, liveAt ?? 0) return { state: 'interrupted', completedAt: Math.min(lastAlive, evidence.observedAt) } } -/** When the latest Stop event found `item`'s turn running: E1 writes one only for a live turn. */ -export function stopFoundTurnLiveAt( - journal: Pick, +/** The latest observation of this turn's owner, including a Stop that found it running. */ +export function lastProvenTurnLiveAt( + journal: Pick, item: AgentJournalRenderItem ): number | undefined { const stop = journal.stopMarks.latest() const turnId = readAgentJournalTurn(item.body)?.turnId - return stop && turnId !== undefined && stop.event.turnId === turnId ? stop.event.at : undefined + const stoppedAt = stop && turnId !== undefined && stop.event.turnId === turnId ? stop.event.at : 0 + const fence = journal.itemFence(item.itemId) + const providerAt = fence === undefined ? 0 : (journal.lastProviderActivityAt(fence) ?? 0) + return Math.max(stoppedAt, providerAt) || undefined } /** Every turn this settle interrupts is a person's Stop's to end (`turnEndAfterStop`), so it reads @@ -106,7 +109,7 @@ export function runningTurnLifecycleRevisions( export function provenUnverifiableTurnRevisions( items: readonly AgentJournalRenderItem[], evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick + journal: Pick ): JournalLifecycleMutationInput[] { const ownerFence = evidence?.ownerFence if (ownerFence === undefined) { @@ -118,7 +121,7 @@ export function provenUnverifiableTurnRevisions( ? turnLifecycleRevision( item, turn, - turnVerdictFromDeathEvidence(evidence, ownerFence, stopFoundTurnLiveAt(journal, item)) + turnVerdictFromDeathEvidence(evidence, ownerFence, lastProvenTurnLiveAt(journal, item)) ) : [] }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 4d099a26394d..67b35d246e41 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -99,6 +99,7 @@ describe('provider-exit settlement', () => { items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })] }), itemFence: () => 7, + lastProviderActivityAt: () => undefined, stopMarks: { latest: () => null, personStopDecides: () => false }, appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []) From 5c221a7828ce11a2ef2eefd1a83cb835d3f3f41f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:38:53 -0700 Subject: [PATCH 02/13] test: run restart duration regression in Node --- config/scripts/vitest-sqlite-runtime-files.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 175abca2c9f7..259146465a99 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -115,6 +115,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts', From 75352be3d1feff5520a1acb2f184db93f81aa263 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:51:50 -0700 Subject: [PATCH 03/13] fix: preserve provider receipt time in journal updates --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + src/main/claude/claude-message-journaling.ts | 16 +- .../claude-streamed-text-checkpoints.test.ts | 18 +- .../claude-streamed-text-checkpoints.ts | 17 +- src/main/claude/claude-streamed-thinking.ts | 2 +- .../claude-structured-journal-translation.ts | 2 +- .../codex-structured-item-stream-contracts.ts | 6 +- .../codex/codex-structured-item-streams.ts | 46 +-- .../codex/codex-structured-journal-items.ts | 7 +- .../codex/codex-structured-journal-sink.ts | 22 ++ .../codex-structured-journal-translation.ts | 7 +- .../agent-session-journal/journal-item-row.ts | 49 +++ .../journal-provider-activity.ts | 9 +- .../agent-session-journal/journal-row-base.ts | 11 + .../journal-row-builders.ts | 63 +--- .../journal-row-schema.test.ts | 42 +++ .../journal-row-schema.ts | 2 + .../journal-row-unusable-annotations.ts | 8 + .../journal-store-contracts.ts | 2 + ...tructured-agent-session-event-sink.test.ts | 35 +- .../structured-agent-session-event-sink.ts | 33 +- .../structured-agent-session-item-append.ts | 27 ++ ...ed-agent-session-journal-append-options.ts | 18 +- ...tructured-agent-session-resolved-append.ts | 13 +- ...nt-session-restart-provider-output.test.ts | 343 ++++++++++++++++++ .../structured-agent-session-transition.ts | 22 +- 26 files changed, 692 insertions(+), 129 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-item-row.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-row-base.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 259146465a99..8a3e6e0468eb 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -113,6 +113,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts', diff --git a/src/main/claude/claude-message-journaling.ts b/src/main/claude/claude-message-journaling.ts index 9d2e00a2e67f..4ff785e368e5 100644 --- a/src/main/claude/claude-message-journaling.ts +++ b/src/main/claude/claude-message-journaling.ts @@ -108,7 +108,10 @@ export function journalClaudeMessage( // output; a reader that scans back to the turn record and stops would // otherwise look straight past the row that opened it. ctx.turn.ensureOpen(message, source, observedAt) - ctx.sink.appendItem(identity, body, stamp(identity, body)) + ctx.sink.appendItem(identity, body, { + ...stamp(identity, body), + providerObservedAt: observedAt + }) changed = true } for (const tool of claudeToolUses(outputEnvelope)) { @@ -125,7 +128,10 @@ export function journalClaudeMessage( } const toolIdentity = claudeToolIdentity(envelope.sessionId, tool.id) const toolBody = claudeToolBody({ tool }) - ctx.sink.appendItem(toolIdentity, toolBody, stamp(toolIdentity, toolBody)) + ctx.sink.appendItem(toolIdentity, toolBody, { + ...stamp(toolIdentity, toolBody), + providerObservedAt: observedAt + }) changed = true } for (const result of results) { @@ -136,7 +142,10 @@ export function journalClaudeMessage( } const resultIdentity = claudeToolIdentity(envelope.sessionId, result.toolUseId) const resultBody = claudeToolBody({ tool, result }) - ctx.sink.appendItem(resultIdentity, resultBody, stamp(resultIdentity, resultBody)) + ctx.sink.appendItem(resultIdentity, resultBody, { + ...stamp(resultIdentity, resultBody), + providerObservedAt: observedAt + }) ctx.subagents.observeToolResult(result.toolUseId, result.failed) if ( results.length === 1 && @@ -154,6 +163,7 @@ export function journalClaudeMessage( // The write that ends the row: shed under pressure, the row would read open for good. ctx.sink.appendItem(thinking.identity, thinking.body, { ...stamp(thinking.identity, thinking.body), + providerObservedAt: observedAt, ...(thinking.startedAt === undefined ? {} : { observedAt: thinking.startedAt }), lifecycle: true }) diff --git a/src/main/claude/claude-streamed-text-checkpoints.test.ts b/src/main/claude/claude-streamed-text-checkpoints.test.ts index 89f825bc5605..ce4df1f7e9c4 100644 --- a/src/main/claude/claude-streamed-text-checkpoints.test.ts +++ b/src/main/claude/claude-streamed-text-checkpoints.test.ts @@ -142,7 +142,7 @@ describe('claude streamed text checkpoints', () => { runWindow() expect(rows).toEqual([{ uuid: 'block-1', text: 'hello' }]) - expect(stamps).toEqual([CHILD_LINKAGE]) + expect(stamps).toEqual([{ ...CHILD_LINKAGE, providerObservedAt: expect.any(Number) }]) }) it("writes no linkage keys for a block the session's own agent streamed", () => { @@ -151,7 +151,7 @@ describe('claude streamed text checkpoints', () => { store.append(identityOf('block-1'), 'hello') runWindow() - expect(stamps).toEqual([{}]) + expect(stamps).toEqual([{ providerObservedAt: expect.any(Number) }]) }) it('writes a checkpoint at once while the producing agent is provisional', () => { @@ -176,7 +176,7 @@ describe('claude streamed text checkpoints', () => { store.reattribute() expect(rows.at(-1)).toEqual({ uuid: 'block-1', text: 'partial' }) - expect(stamps.at(-1)).toEqual(CHILD_LINKAGE) + expect(stamps.at(-1)).toEqual({ ...CHILD_LINKAGE, providerObservedAt: expect.any(Number) }) }) it('writes a held block under the raw reference when no announcement comes', () => { @@ -190,7 +190,12 @@ describe('claude streamed text checkpoints', () => { expect(rows).toEqual([{ uuid: 'block-1', text: 'never announced' }]) expect(stamps).toEqual([ - { agentId: 'toolu_1', providerParentRef: 'toolu_1', producerKind: 'agent' } + { + agentId: 'toolu_1', + providerParentRef: 'toolu_1', + producerKind: 'agent', + providerObservedAt: expect.any(Number) + } ]) // Nothing ever names it, so re-attribution has nothing better to say and @@ -213,7 +218,10 @@ describe('claude streamed text checkpoints', () => { store.append(identityOf('block-1'), 'first and more', 'toolu_1') store.flush() - expect(stamps).toEqual([CHILD_LINKAGE, { ...CHILD_LINKAGE, agentId: 'task-2' }]) + expect(stamps).toEqual([ + { ...CHILD_LINKAGE, providerObservedAt: expect.any(Number) }, + { ...CHILD_LINKAGE, agentId: 'task-2', providerObservedAt: expect.any(Number) } + ]) }) it('stops persisting once disposed', () => { diff --git a/src/main/claude/claude-streamed-text-checkpoints.ts b/src/main/claude/claude-streamed-text-checkpoints.ts index cf8d3c40be5b..ee9d113a20f9 100644 --- a/src/main/claude/claude-streamed-text-checkpoints.ts +++ b/src/main/claude/claude-streamed-text-checkpoints.ts @@ -31,7 +31,8 @@ export type ClaudeStreamedTextCheckpoints = { append: ( identity: AgentJournalItemIdentity, text: string, - parentToolUseId?: string | null + parentToolUseId?: string | null, + providerObservedAt?: number ) => void /** Write every block whose row is behind the text received for it. */ flush: () => void @@ -90,16 +91,19 @@ export function createClaudeStreamedTextCheckpoints( const writtenLinkage = new Map() const latestText = new Map() const checkpointLengths = new Map() + const observations = new Map() /** Resolved FRESH on every checkpoint. A provisional producer is stamped with * the handle it has rather than holding the prose back: the next checkpoint, * or `reattribute` once the announcement lands, revises the same row. */ const producerOptions = (key: string): StructuredAgentSessionAppendOptions => { const scope = scopes.get(key) ?? null - if (scope === null) { - return {} + return { + ...(scope === null + ? {} + : agentJournalLinkageFields(deps.producer.settledLinkageFor(scope).linkage)), + providerObservedAt: observations.get(key) } - return agentJournalLinkageFields(deps.producer.settledLinkageFor(scope).linkage) } const persist = (key: string, text: string, force: boolean): void => { @@ -132,13 +136,15 @@ export function createClaudeStreamedTextCheckpoints( writtenLinkage.delete(key) latestText.delete(key) checkpointLengths.delete(key) + observations.delete(key) } return { - append: (identity, text, parentToolUseId = null) => { + append: (identity, text, parentToolUseId = null, providerObservedAt = Date.now()) => { const key = agentJournalItemKey(identity) identities.set(key, identity) scopes.set(key, parentToolUseId) + observations.set(key, providerObservedAt) coalescer.append(key, text) }, flush: () => { @@ -196,6 +202,7 @@ export function createClaudeStreamedTextCheckpoints( writtenLinkage.clear() latestText.clear() checkpointLengths.clear() + observations.clear() } } } diff --git a/src/main/claude/claude-streamed-thinking.ts b/src/main/claude/claude-streamed-thinking.ts index 1edd109adebf..431cca283c4b 100644 --- a/src/main/claude/claude-streamed-thinking.ts +++ b/src/main/claude/claude-streamed-thinking.ts @@ -109,7 +109,7 @@ export function createClaudeStreamedThinking(deps: { startedAt: delta.startedAt }) } - checkpoints.append(identity, delta.text, delta.parentToolUseId) + checkpoints.append(identity, delta.text, delta.parentToolUseId, observedAt) return true }, /** The row a final frame's thinking lands on — its streamed block's, else its own — closed. diff --git a/src/main/claude/claude-structured-journal-translation.ts b/src/main/claude/claude-structured-journal-translation.ts index e2932e0d3468..6b5dc5bef691 100644 --- a/src/main/claude/claude-structured-journal-translation.ts +++ b/src/main/claude/claude-structured-journal-translation.ts @@ -168,7 +168,7 @@ export function createClaudeJournalTranslator( delta || thinking ? claudeStreamTurnSource(message) : claudeStreamTurnStartSource(message) turn.ensureOpen(message, source, observedAt) if (delta) { - streamedText.append(delta.identity, delta.text, delta.parentToolUseId) + streamedText.append(delta.identity, delta.text, delta.parentToolUseId, observedAt) } return delta !== null || thinking } diff --git a/src/main/codex/codex-structured-item-stream-contracts.ts b/src/main/codex/codex-structured-item-stream-contracts.ts index 38b9a9d3038a..6b64b2bcc892 100644 --- a/src/main/codex/codex-structured-item-stream-contracts.ts +++ b/src/main/codex/codex-structured-item-stream-contracts.ts @@ -19,6 +19,7 @@ export type CodexItemStreamDeps = { maxTotalRetainedBytes?: number maxMetadataBytes?: number schedule?: AgentSessionDeltaCoalescerDeps['schedule'] + now?: () => number } export type CodexItemStreamState = { @@ -26,11 +27,13 @@ export type CodexItemStreamState = { item: CodexThreadItem /** Host clock when the item started, for a row its stream writes first. */ startedAt?: number + providerObservedAt?: number } export type CodexPendingItemPatch = { identity: AgentJournalItemIdentity body: NonNullable['body']> + providerObservedAt: number } export type CodexStructuredItemStreamAdmission = @@ -55,7 +58,8 @@ export type CodexStructuredItemStreams = { handle: ( threadId: string, method: string, - params: unknown + params: unknown, + observedAt?: number ) => CodexStructuredItemStreamHandleResult forget: (threadId: string, itemId: string) => void flush: () => boolean diff --git a/src/main/codex/codex-structured-item-streams.ts b/src/main/codex/codex-structured-item-streams.ts index c783458ca298..ae47ba361e23 100644 --- a/src/main/codex/codex-structured-item-streams.ts +++ b/src/main/codex/codex-structured-item-streams.ts @@ -4,9 +4,11 @@ import { } from '../../shared/agent-session-journal-types' import { createAgentSessionDeltaCoalescer } from '../native-chat/agent-session-wire/agent-session-delta-coalescer' import { CodexItemStreamRetention } from './codex-item-stream-retention' -import { appendCodexItemAndPublish } from './codex-structured-journal-sink' -import { codexJournalItem, codexStreamingJournalItem } from './codex-structured-item-translation' -import { withJournalReasoningLifecycle } from '../native-chat/agent-session-journal/journal-reasoning-row' +import { + appendCodexItemAndPublish, + appendCodexStreamCheckpoint +} from './codex-structured-journal-sink' +import { codexJournalItem } from './codex-structured-item-translation' import { codexStructuredItemKey, MAX_CODEX_ITEM_STREAM_PENDING_PATCHES, @@ -109,19 +111,6 @@ export function createCodexStructuredItemStreams( } } - const append = (key: string, state: CodexItemStreamState, text: string): boolean => { - const translated = codexStreamingJournalItem(state.item, text) - if (!translated.body) { - return true - } - // A stream only ever carries an item that has not completed yet. - const body = withJournalReasoningLifecycle(translated.body, { state: 'running' }) - return appendCodexItemAndPublish(deps.sink, state.identity, body, { - ...attributionOf(key), - ...(state.startedAt === undefined ? {} : { observedAt: state.startedAt }) - }).accepted - } - const shouldPersist = (key: string, textLength: number): boolean => { const checkpointLength = checkpointLengths.get(key) ?? 0 const nextLength = Math.max(checkpointLength + 32, Math.ceil(checkpointLength * 1.125)) @@ -133,7 +122,7 @@ export function createCodexStructuredItemStreams( return true } const state = states.get(key) - if (state && append(key, state, text)) { + if (state && appendCodexStreamCheckpoint(deps.sink, attributionOf(key), state, text)) { checkpointLengths.set(key, text.length) pendingCheckpoints.delete(key) return true @@ -196,12 +185,10 @@ export function createCodexStructuredItemStreams( if (!pending) { return { accepted: true } } - const admission = appendCodexItemAndPublish( - deps.sink, - pending.identity, - pending.body, - attributionOf(key) - ) + const admission = appendCodexItemAndPublish(deps.sink, pending.identity, pending.body, { + ...attributionOf(key), + providerObservedAt: pending.providerObservedAt + }) if (!admission.accepted) { return admission } @@ -221,14 +208,19 @@ export function createCodexStructuredItemStreams( ), track: (threadId, turnId, item, identity, startedAt) => { const key = codexStructuredItemKey(threadId, item.id) - if (!states.retain(key, codexItemStreamState(item, identity, startedAt))) { + if ( + !states.retain(key, { + ...codexItemStreamState(item, identity, startedAt), + providerObservedAt: startedAt ?? Date.now() + }) + ) { return false } producers.set(key, { threadId, turnId }) trimStates() return true }, - handle: (threadId, method, params) => { + handle: (threadId, method, params, observedAt) => { const paramsRecord = readCodexItemStreamRecord(params) const itemId = readCodexItemStreamString(paramsRecord, 'itemId') if (method === PATCH_UPDATED_METHOD) { @@ -251,7 +243,8 @@ export function createCodexStructuredItemStreams( if (translated.body) { const nextPending: CodexPendingItemPatch = { identity: state.identity, - body: translated.body + body: translated.body, + providerObservedAt: observedAt ?? deps.now?.() ?? Date.now() } const previous = pendingPatches.get(key) const previousBytes = previous ? pendingPatchBytes(previous) : 0 @@ -291,6 +284,7 @@ export function createCodexStructuredItemStreams( const delta = method === REASONING_PART_METHOD ? '\n' : paramsRecord.delta if (typeof delta === 'string') { const key = codexStructuredItemKey(threadId, state.item.id) + state.providerObservedAt = observedAt ?? deps.now?.() ?? Date.now() pendingCheckpoints.add(key) const accepted = coalescer.append(key, delta) if (!accepted) { diff --git a/src/main/codex/codex-structured-journal-items.ts b/src/main/codex/codex-structured-journal-items.ts index 17a002225e7c..3a5a9066e53d 100644 --- a/src/main/codex/codex-structured-journal-items.ts +++ b/src/main/codex/codex-structured-journal-items.ts @@ -64,6 +64,7 @@ export class CodexJournalItems { coalesceMs: deps.coalesceMs, maxRetainedBytes: deps.maxRetainedBytes, schedule: deps.schedule, + now: deps.now, maxMetadataBytes: deps.maxMetadataBytes, turnIdFor: (threadId, params) => readCodexTurnId(params) ?? this.activeTurn(threadId), identityFor: (threadId, turnId, item) => this.identityFor(threadId, turnId, item), @@ -151,6 +152,7 @@ export class CodexJournalItems { const startedAt = event.method === 'item/completed' ? active?.startedAt : receivedAt const admission = this.appendTranslated(event.method, identity, translated, { ...this.deps.attributionFor(event.threadId, turnId), + providerObservedAt: receivedAt, ...(startedAt === undefined ? {} : { observedAt: startedAt }) }) if (!admission.accepted) { @@ -180,7 +182,10 @@ export class CodexJournalItems { method: string, identity: AgentJournalItemIdentity, translated: ReturnType, - attribution: AgentJournalRowAttribution & { observedAt?: number } + attribution: AgentJournalRowAttribution & { + observedAt?: number + providerObservedAt?: number + } ): CodexJournalTranslationAdmission { if (!translated.body) { return CODEX_JOURNAL_ADMITTED diff --git a/src/main/codex/codex-structured-journal-sink.ts b/src/main/codex/codex-structured-journal-sink.ts index dc4778ea3501..ae573a903c12 100644 --- a/src/main/codex/codex-structured-journal-sink.ts +++ b/src/main/codex/codex-structured-journal-sink.ts @@ -1,3 +1,6 @@ +import type { CodexItemStreamState } from './codex-structured-item-stream-contracts' +import { codexStreamingJournalItem } from './codex-structured-item-translation' +import { withJournalReasoningLifecycle } from '../native-chat/agent-session-journal/journal-reasoning-row' import type { AgentJournalItemBody, AgentJournalItemIdentity, @@ -164,3 +167,22 @@ export function admitCodexLifecycleItems( } return publishCodexLifecycle(sink) } + +export function appendCodexStreamCheckpoint( + sink: StructuredAgentSessionEventSink, + attribution: AgentJournalRowAttribution, + state: CodexItemStreamState, + text: string +): boolean { + const translated = codexStreamingJournalItem(state.item, text) + if (!translated.body) { + return true + } + // A stream only ever carries an item that has not completed yet. + const body = withJournalReasoningLifecycle(translated.body, { state: 'running' }) + return appendCodexItemAndPublish(sink, state.identity, body, { + ...attribution, + providerObservedAt: state.providerObservedAt, + ...(state.startedAt === undefined ? {} : { observedAt: state.startedAt }) + }).accepted +} diff --git a/src/main/codex/codex-structured-journal-translation.ts b/src/main/codex/codex-structured-journal-translation.ts index 19fee37a5368..1cc7088d4b2b 100644 --- a/src/main/codex/codex-structured-journal-translation.ts +++ b/src/main/codex/codex-structured-journal-translation.ts @@ -188,7 +188,12 @@ export function createCodexJournalTranslator( return CODEX_JOURNAL_ADMITTED } if (event.type === 'notification') { - const streamResult = items.streams.handle(event.threadId, event.method, event.params) + const streamResult = items.streams.handle( + event.threadId, + event.method, + event.params, + event.observedAt + ) if (streamResult.handled) { return publishActivity(event, streamResult.admission) } diff --git a/src/main/native-chat/agent-session-journal/journal-item-row.ts b/src/main/native-chat/agent-session-journal/journal-item-row.ts new file mode 100644 index 000000000000..d673c0dd84d0 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-item-row.ts @@ -0,0 +1,49 @@ +import type { + AgentJournalItemBody, + AgentJournalItemIdentity, + AgentJournalProducerLinkage, + AgentJournalTurnScope +} from '../../../shared/agent-session-journal-types' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' +import type { JournalReducerState } from './journal-reducer' +import type { JournalItemRow } from './journal-row-schema' +import { journalRowBase } from './journal-row-base' +import { turnEndAfterStop } from './journal-stop-turn-end' + +export function buildJournalItemRow(input: { + state: JournalReducerState + identity: AgentJournalItemIdentity + body: AgentJournalItemBody + seq: number + fence: number + ts: number + recovered?: true + providerObservedAt?: number + linkage?: AgentJournalProducerLinkage + turnScope: AgentJournalTurnScope +}): JournalItemRow { + const itemId = agentJournalItemKey(input.identity) + const resolved = input.state.aliases.get(itemId) ?? itemId + // A tombstoned row keeps its revision in `tombstones`, and the reducer drops + // any item at or below it — so a re-add has to outrank the tombstone too. + const revision = + Math.max( + input.state.items.get(resolved)?.revision ?? 0, + input.state.tombstones.get(resolved) ?? 0 + ) + 1 + const body = turnEndAfterStop(input.state, resolved, input.body) + return { + kind: 'item', + itemId, + revision, + body, + ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), + ...(input.recovered ? { recovered: input.recovered } : {}), + ...(input.providerObservedAt === undefined + ? {} + : { providerObservedAt: input.providerObservedAt }), + turnScope: input.turnScope, + ...agentJournalLinkageFields(input.linkage) + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-provider-activity.ts b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts index 7c778210e31a..260fb6397692 100644 --- a/src/main/native-chat/agent-session-journal/journal-provider-activity.ts +++ b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts @@ -15,18 +15,18 @@ export function observeJournalProviderActivity( return } const identity = parseAgentJournalItemKey(itemId) - if (!identity || identity.provider === 'orca') { + if (!identity || (identity.provider === 'orca' && row.providerObservedAt === undefined)) { return } state.providerActivityAt.set( row.fence, - Math.max(state.providerActivityAt.get(row.fence) ?? 0, row.ts) + Math.max(state.providerActivityAt.get(row.fence) ?? 0, row.providerObservedAt ?? row.ts) ) } function isProviderActivity(body: AgentJournalItemBody): boolean { if (body.kind === 'message') { - return body.role === 'assistant' + return body.role !== 'user' } if (body.kind === 'tool-call') { return body.endedAs === undefined @@ -34,5 +34,6 @@ function isProviderActivity(body: AgentJournalItemBody): boolean { if (body.kind === 'approval' || body.kind === 'question') { return body.resolution.state === 'pending' } - return readAgentJournalTurn(body)?.state === 'running' + const turn = readAgentJournalTurn(body) + return turn ? turn.state === 'running' : true } diff --git a/src/main/native-chat/agent-session-journal/journal-row-base.ts b/src/main/native-chat/agent-session-journal/journal-row-base.ts new file mode 100644 index 000000000000..2eb00979c936 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-row-base.ts @@ -0,0 +1,11 @@ +import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' + +export function journalRowBase( + epoch: string, + seq: number, + fence: number, + ts: number, + bodies: readonly { kind: string }[] = [] +): { v: number; epoch: string; seq: number; fence: number; ts: number } { + return { v: journalRowSchemaVersion(bodies), epoch, seq, fence, ts } +} diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index 27951b308e92..a8187e95d54d 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -10,7 +10,6 @@ import type { } from '../../../shared/agent-session-journal-types' import type { AgentSessionMessageSource } from '../../../shared/agent-session-message-source' import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' -import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import { agentJournalLinkageFields, namesAgentJournalProducer @@ -33,6 +32,10 @@ import { boundInlineText, DEFAULT_JOURNAL_PAYLOAD_LIMITS } from './journal-paylo import { assertSubmissionIdUnused } from './journal-write-guards' import { turnEndAfterStop } from './journal-stop-turn-end' import type { ResolveDispatchInput } from './journal-store-contracts' +import { journalRowBase } from './journal-row-base' +import { buildJournalItemRow } from './journal-item-row' +export { journalRowBase } from './journal-row-base' +export { buildJournalItemRow } from './journal-item-row' type RowBuilder = (seq: number, ts: number) => T @@ -40,7 +43,12 @@ export function journalItemRowBuilder( state: () => JournalReducerState, identity: AgentJournalItemIdentity, body: AgentJournalItemBody, - options: AgentJournalRowAttribution & { fence: number; observedAt?: number; recovered?: true } + options: AgentJournalRowAttribution & { + fence: number + observedAt?: number + providerObservedAt?: number + recovered?: true + } ): RowBuilder { return (seq, ts) => buildJournalItemRow({ @@ -50,6 +58,7 @@ export function journalItemRowBuilder( seq, fence: options.fence, ts: options.observedAt ?? ts, + providerObservedAt: options.providerObservedAt, recovered: options.recovered, linkage: options, turnScope: options.turnScope @@ -200,7 +209,7 @@ export function journalLifecycleBatchRowBuilder( * An item mutation names its own, or none to keep the row's existing one. * The reducer still reads row-level linkage as the fallback for a mutation * that names none, because a row may come from a host that wrote one. */ - options: { fence: number; recovered?: true } + options: { fence: number; recovered?: true; providerObservedAt?: number } ): RowBuilder { return (seq, ts) => { if (mutations.length === 0 || mutations.length > MAX_JOURNAL_LIFECYCLE_BATCH_MUTATIONS) { @@ -237,7 +246,10 @@ export function journalLifecycleBatchRowBuilder( ts, built.flatMap((mutation) => (mutation.kind === 'item' ? [mutation.body] : [])) ), - ...(options.recovered ? { recovered: options.recovered } : {}) + ...(options.recovered ? { recovered: options.recovered } : {}), + ...(options.providerObservedAt === undefined + ? {} + : { providerObservedAt: options.providerObservedAt }) } if (Buffer.byteLength(JSON.stringify(row), 'utf8') + 1 > MAX_JOURNAL_LIFECYCLE_BATCH_BYTES) { throw new Error('journal_lifecycle_batch_byte_bound_exceeded') @@ -246,49 +258,6 @@ export function journalLifecycleBatchRowBuilder( } } -export function journalRowBase( - epoch: string, - seq: number, - fence: number, - ts: number, - bodies: readonly { kind: string }[] = [] -): { v: number; epoch: string; seq: number; fence: number; ts: number } { - return { v: journalRowSchemaVersion(bodies), epoch, seq, fence, ts } -} - -export function buildJournalItemRow(input: { - state: JournalReducerState - identity: AgentJournalItemIdentity - body: AgentJournalItemBody - seq: number - fence: number - ts: number - recovered?: true - linkage?: AgentJournalProducerLinkage - turnScope: AgentJournalTurnScope -}): JournalItemRow { - const itemId = agentJournalItemKey(input.identity) - const resolved = input.state.aliases.get(itemId) ?? itemId - // A tombstoned row keeps its revision in `tombstones`, and the reducer drops - // any item at or below it — so a re-add has to outrank the tombstone too. - const revision = - Math.max( - input.state.items.get(resolved)?.revision ?? 0, - input.state.tombstones.get(resolved) ?? 0 - ) + 1 - const body = turnEndAfterStop(input.state, resolved, input.body) - return { - kind: 'item', - itemId, - revision, - body, - ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), - ...(input.recovered ? { recovered: input.recovered } : {}), - turnScope: input.turnScope, - ...agentJournalLinkageFields(input.linkage) - } -} - export function buildJournalTombstoneRow(input: { state: JournalReducerState itemId: string diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts index b72a24d70bb3..93a3bc7b4ec6 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts @@ -22,6 +22,48 @@ function parse(row: Record): boolean { } describe('journal row validation', () => { + it.each([null, 'later', -1, {}])( + 'drops unusable provider receipt %j without dropping output', + (providerObservedAt) => { + const parsed = parseJournalRow( + JSON.stringify({ + ...BASE, + kind: 'item', + itemId: 'i-1', + revision: 1, + providerObservedAt, + body: { kind: 'status', text: 'provider output' } + }) + ) + expect(parsed).toEqual({ + ok: true, + row: { + ...BASE, + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + kind: 'item', + itemId: 'i-1', + revision: 1, + body: { kind: 'status', text: 'provider output' } + } + }) + } + ) + + it('keeps a valid optional receipt on a same-version row', () => { + const row = { + ...BASE, + kind: 'item', + itemId: 'i-1', + revision: 1, + providerObservedAt: 5_500, + body: { kind: 'status', text: 'provider output' } + } + expect(parseJournalRow(JSON.stringify(row))).toEqual({ + ok: true, + row: { ...row, v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION } + }) + }) + it('upcasts v1 rows to the current schema without changing their body', () => { const parsed = parseJournalRow( JSON.stringify({ diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index 1d226761ae25..f647c073b3e7 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -55,6 +55,8 @@ type JournalRowBase = AgentJournalProducerLinkage & { fence: number /** Observed (provider or host) timestamp. Ordering is by `seq`, not by this. */ ts: number + /** Receipt of live provider output, distinct from an item's pinned start timestamp. */ + providerObservedAt?: number /** Set when crash reconciliation appended the row after the fact. */ recovered?: true /** Which turn the item this row creates belongs to. Rides the base, and is not a `v` bump, diff --git a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts index 6c8811eb4a5b..130226cf14e1 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts @@ -6,6 +6,14 @@ import { isAdmissibleAgentSessionContextUsage } from '../../../shared/agent-sess export function dropUnusableRowAnnotations(record: Record): void { dropUnusableProducerLinkage(record) dropUnusableTurnScope(record) + if ( + record.providerObservedAt !== undefined && + (typeof record.providerObservedAt !== 'number' || + !Number.isFinite(record.providerObservedAt) || + record.providerObservedAt < 0) + ) { + delete record.providerObservedAt + } if (record.kind === 'lifecycle-batch' && Array.isArray(record.mutations)) { for (const mutation of record.mutations) { if (isPlainObject(mutation)) { diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 41b892adf348..c41fcec55322 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -57,6 +57,7 @@ export type JournalAppendResult = { export type JournalItemAppendOptions = AgentJournalRowAttribution & { fence: number observedAt?: number + providerObservedAt?: number recovered?: true } export type JournalTombstoneInput = { fence: number } @@ -74,6 +75,7 @@ export type JournalLifecycleBatchInput = { mutations: readonly JournalLifecycleMutationInput[] fence: number recovered?: true + providerObservedAt?: number /** Rejects the sends still queued with this first, in the same append: a failed start's row * follows the messages it failed, and no reader meets one without the other. With none still * queued, the batch is not written either. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts index 476089292a93..a0f2d8485a0d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts @@ -120,6 +120,23 @@ function target( } describe('deferred structured agent-session event sink', () => { + it('records receipt before a deferred append waits for its journal', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) + try { + deferred.sink.appendItem(identity(1), BODY, { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) + now.mockReturnValue(9_000) + deferred.bind(target(5, [])) + await deferred.drained() + expect(journalAppendOptions).toEqual([ + { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, providerObservedAt: 1_000 } + ]) + } finally { + deferred.close() + now.mockRestore() + } + }) + it('buffers writes made before the journal exists and drains them in arrival order', async () => { const log: Recorded[] = [] const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) @@ -577,7 +594,12 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } + { + fence: 5, + turnScope: AGENT_JOURNAL_THREAD_SCOPE, + ...LINKAGE, + providerObservedAt: expect.any(Number) + } ]) deferred.close() } @@ -601,7 +623,12 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } + { + fence: 5, + turnScope: AGENT_JOURNAL_THREAD_SCOPE, + ...LINKAGE, + providerObservedAt: expect.any(Number) + } ]) deferred.close() } @@ -637,7 +664,9 @@ describe('producer linkage reaches the journal through every append path', () => // A control, not a pin. Absence is the claim, so the keys must be missing // rather than present-and-undefined: a reader holding this options object // would read `agentId: undefined` as a key that exists. - expect(journalAppendOptions).toEqual([{ fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE }]) + expect(journalAppendOptions).toEqual([ + { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, providerObservedAt: expect.any(Number) } + ]) deferred.close() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index 4a09b9fcacc9..d3b662d72710 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -8,9 +8,9 @@ import type { import type { AgentSessionTurnActivity } from '../../../shared/agent-session-wire' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' -import { estimateStructuredAgentSessionItemBytes } from './structured-agent-session-event-sink-estimate' import { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' -import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' +import { providerObservedAppendOptions } from './structured-agent-session-journal-append-options' +import { createStructuredAgentSessionItemAppend } from './structured-agent-session-item-append' import { createStructuredAgentSessionResolvedAppend } from './structured-agent-session-resolved-append' import { createStructuredAgentSessionTransitionMembers, @@ -38,6 +38,8 @@ export type StructuredAgentSessionAppendOptions = AgentJournalProducerLinkage & lifecycle?: boolean /** Host clock to stamp on the row instead of its append time. */ observedAt?: number + /** Latest frame receipt; a coalesced checkpoint keeps this rather than its write time. */ + providerObservedAt?: number } export type StructuredAgentSessionPublishOptions = Pick< @@ -237,8 +239,9 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { settlementId: string, mutations: readonly JournalLifecycleMutationInput[], options: StructuredAgentSessionAppendOptions = {} - ): StructuredAgentSessionSinkAdmission => - queue.submit( + ): StructuredAgentSessionSinkAdmission => { + const observation = providerObservedAppendOptions(options) + return queue.submit( { bytes: Buffer.byteLength(JSON.stringify({ settlementId, mutations }), 'utf8') + 512, run: (bound) => @@ -246,11 +249,13 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { settlementId, mutations, // No row-level linkage: each mutation names its own (see the batch row builder). - fence: bound.fence + fence: bound.fence, + providerObservedAt: observation.providerObservedAt }) }, { ...options, lifecycle: true } ) + } const publish = ( options: StructuredAgentSessionPublishOptions = {} @@ -264,23 +269,7 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { options ) - const appendItem: NonNullable = ( - identity, - body, - options - ) => - queue.submit( - { - bytes: estimateStructuredAgentSessionItemBytes(identity, body), - run: (bound) => - bound.journal.appendItem( - identity, - body, - structuredAgentSessionJournalAppendOptions(bound.fence, options) - ) - }, - options - ) + const appendItem = createStructuredAgentSessionItemAppend(queue) return { sink: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts new file mode 100644 index 000000000000..906eb8d1c302 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts @@ -0,0 +1,27 @@ +import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' +import { estimateStructuredAgentSessionItemBytes } from './structured-agent-session-event-sink-estimate' +import { + providerObservedAppendOptions, + structuredAgentSessionJournalAppendOptions +} from './structured-agent-session-journal-append-options' + +export function createStructuredAgentSessionItemAppend( + queue: StructuredAgentSessionSinkQueue +): NonNullable { + return (identity, body, options) => { + const observation = providerObservedAppendOptions(options) + return queue.submit( + { + bytes: estimateStructuredAgentSessionItemBytes(identity, body), + run: (bound) => + bound.journal.appendItem( + identity, + body, + structuredAgentSessionJournalAppendOptions(bound.fence, observation) + ) + }, + options + ) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts index e7ce667ed2b6..3faf11db4b43 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts @@ -6,7 +6,20 @@ import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import type { JournalItemAppendOptions } from '../agent-session-journal/journal-store-contracts' -import type { StructuredAgentSessionItemAppendOptions } from './structured-agent-session-event-sink' +import type { + StructuredAgentSessionAppendOptions, + StructuredAgentSessionItemAppendOptions +} from './structured-agent-session-event-sink' + +/** Capture before admission, so a delayed journal write cannot extend provider life. */ +export function providerObservedAppendOptions( + options: T +): T & { providerObservedAt: number } { + return { + ...options, + providerObservedAt: options.providerObservedAt ?? options.observedAt ?? Date.now() + } +} export function structuredAgentSessionJournalAppendOptions( fence: number, @@ -15,6 +28,9 @@ export function structuredAgentSessionJournalAppendOptions( return { fence, ...(options.observedAt === undefined ? {} : { observedAt: options.observedAt }), + ...(options.providerObservedAt === undefined + ? {} + : { providerObservedAt: options.providerObservedAt }), turnScope: options.turnScope, ...agentJournalLinkageFields(options) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts index d3640db2a0cf..569fbdf73b54 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts @@ -8,7 +8,10 @@ import type { StructuredAgentSessionEventSink, StructuredAgentSessionRevisionJournal } from './structured-agent-session-event-sink' -import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' +import { + providerObservedAppendOptions, + structuredAgentSessionJournalAppendOptions +} from './structured-agent-session-journal-append-options' import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' type ResolvedItem = { identity: AgentJournalItemIdentity; body: AgentJournalItemBody } @@ -32,8 +35,9 @@ export function createStructuredAgentSessionResolvedAppend( resolve: (journal: StructuredAgentSessionRevisionJournal) => ResolvedItem | null, options: StructuredAgentSessionItemAppendOptions, input: { publish: boolean; lifecycle?: true; overflow: string } - ) => - queue.submit( + ) => { + const observation = providerObservedAppendOptions(options) + return queue.submit( { bytes: reservedBytes, ...(input.lifecycle ? { lifecycle: true } : {}), @@ -51,7 +55,7 @@ export function createStructuredAgentSessionResolvedAppend( } return resolved }, - structuredAgentSessionJournalAppendOptions(bound.fence, options) + structuredAgentSessionJournalAppendOptions(bound.fence, observation) ) if (landed !== null && input.publish) { bound.publish() @@ -60,6 +64,7 @@ export function createStructuredAgentSessionResolvedAppend( }, input.lifecycle ? { lifecycle: true } : options ) + } const ITEM_OVERFLOW = 'structured agent-session resolved item exceeded its reserved size' const identityOnly = (publish: boolean) => ((identitySizeBound, body, resolveIdentity, options) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts new file mode 100644 index 000000000000..bd132c2759fb --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts @@ -0,0 +1,343 @@ +import { createCodexJournalTranslator } from '../../codex/codex-structured-journal-translation' +import { codexStreamingJournalItem } from '../../codex/codex-structured-item-translation' +import { createClaudeJournalTranslator } from '../../claude/claude-structured-journal-translation' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + completedStructuredAgentTurnSeconds, + selectStructuredAgentTurnTimings +} from '../../../shared/structured-agent-session-turn-timing' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { HOST_TEST_NOW, hostTestAttachParams } from './structured-agent-session-host-test-data' +import { + createRestTestRig, + REST_TEST_CALLER, + restTestSend, + REST_TEST_SESSION as SESSION, + REST_TEST_THREAD as THREAD, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 +const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 +const TURN = { provider: 'codex' as const, threadId: THREAD, turnId: 'stream-turn', ordinal: 0 } +const REPLY = { ...TURN, ordinal: 1 } +const TURN_SCOPE = { kind: 'turn' as const, turnItemId: agentJournalItemKey(TURN) } + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await rig.dispose() + closeTestJournalHostDatabases() +}) + +/** Drops the old host without an exit callback or graceful turn settlement. */ +async function restartAfterCrash(): Promise { + const old = rig.host + old.stopDelivery() + const { lifetime, runtimeState, sessions } = old.collaboratorsForTests() + lifetime.idleSweep.dispose() + await runtimeState.stopLeaseRenewal() + runtimeState.currentEventSink(SESSION)?.close() + await sessions.get(SESSION)?.journal.close() + closeTestJournalHostDatabases() + rig.store = await openTestAgentSessionRecordStore(rig.root) + rig.host = new StructuredAgentSessionHost({ + ...old.deps, + store: rig.store, + journalDatabase: openTestJournalHostDatabase(rig.root), + now: () => RESTARTED_AT + }) +} + +async function beginTurn() { + const attached = await rig.host.attach(REST_TEST_CALLER, hostTestAttachParams(null)) + if (!attached.ok) { + throw new Error('attach refused') + } + const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('review', attached.fence)) + if (!sent.ok) { + throw new Error('send refused') + } + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) + const events = rig.adapter.acquire.mock.calls[0]?.[0].events + if (!events) { + throw new Error('event sink absent') + } + events.appendItem( + TURN, + { kind: 'turn', turnId: TURN.turnId, state: 'running', startedAt: HOST_TEST_NOW }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE, observedAt: HOST_TEST_NOW } + ) + return events +} + +async function expectInterruptedOutput(): Promise { + await restartAfterCrash() + await rig.host.restoreReadableSessions() + const snapshot = await rig.host.journalSnapshot(SESSION) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) +} + +describe('provider output survives host restart', () => { + it.each([ + ['agentMessage', 'item/agentMessage/delta'], + ['fileChange', 'item/fileChange/outputDelta'], + ['plan', 'item/plan/delta'], + ['reasoning', 'item/reasoning/textDelta'], + ['commandExecution', 'item/commandExecution/outputDelta'] + ] as const)('times actual Codex %s deltas rather than item start', async (type, method) => { + const events = await beginTurn() + let receivedAt = HOST_TEST_NOW + vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) + const translator = createCodexJournalTranslator({ + sink: events, + primaryThreadId: () => THREAD, + now: () => receivedAt, + schedule: () => () => {} + }) + translator.handle({ + type: 'notification', + sessionId: SESSION, + threadId: THREAD, + method: 'turn/started', + params: { turn: { id: TURN.turnId } }, + observedAt: receivedAt + }) + translator.handle({ + type: 'notification', + sessionId: SESSION, + threadId: THREAD, + method: 'item/started', + params: { + turnId: TURN.turnId, + item: { + id: 'actual-reply', + type, + text: '', + command: 'echo stream', + status: 'inProgress', + changes: [{ path: 'file.ts' }] + } + }, + observedAt: receivedAt + }) + for (let chunk = 1; chunk <= 11; chunk += 1) { + receivedAt = HOST_TEST_NOW + chunk * 500 + expect( + translator.handle({ + type: 'notification', + sessionId: SESSION, + threadId: THREAD, + method, + params: { + turnId: TURN.turnId, + itemId: 'actual-reply', + delta: `actual chunk ${chunk} `.repeat(4) + }, + observedAt: receivedAt + }) + ).toMatchObject({ accepted: true }) + translator.flush() + await rig.host.flushStreamedEvents(SESSION) + } + translator.dispose() + const before = await rig.host.journalSnapshot(SESSION) + const output = before.items.find((item) => + JSON.stringify(item.body).includes('actual chunk 11') + ) + expect(output?.observedAt).toBe(HOST_TEST_NOW) + await restartAfterCrash() + await rig.host.restoreReadableSessions() + const snapshot = await rig.host.journalSnapshot(SESSION) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) + }) + + it('counts streamed file-change output as later proof of provider life', async () => { + const events = await beginTurn() + for (let chunk = 1; chunk <= 11; chunk += 1) { + const translated = codexStreamingJournalItem( + { id: 'file-change-1', type: 'fileChange', changes: [{ path: 'file.ts' }] }, + `patch chunk ${chunk}` + ) + if (!translated.body) { + throw new Error('file change was not translated') + } + events.appendItem(REPLY, translated.body, { + turnScope: TURN_SCOPE, + observedAt: HOST_TEST_NOW + chunk * 500 + }) + await rig.host.flushStreamedEvents(SESSION) + } + expect(rig.store.getRecord(SESSION)?.lease.lastRenewedAt).toBe(HOST_TEST_NOW) + await restartAfterCrash() + await rig.host.restoreReadableSessions() + const snapshot = await rig.host.journalSnapshot(SESSION) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) + }) + + it('keeps a delayed first checkpoint at item start without counting a later flush', async () => { + const events = await beginTurn() + const translator = createCodexJournalTranslator({ + sink: events, + primaryThreadId: () => THREAD, + schedule: () => () => {} + }) + translator.handle({ + type: 'notification', + sessionId: SESSION, + threadId: THREAD, + method: 'turn/started', + params: { turn: { id: TURN.turnId } }, + observedAt: HOST_TEST_NOW + }) + translator.handle({ + type: 'notification', + sessionId: SESSION, + threadId: THREAD, + method: 'item/started', + params: { turnId: TURN.turnId, item: { id: 'late-first', type: 'agentMessage', text: '' } }, + observedAt: HOST_TEST_NOW + }) + translator.handle({ + type: 'notification', + sessionId: SESSION, + threadId: THREAD, + method: 'item/agentMessage/delta', + params: { turnId: TURN.turnId, itemId: 'late-first', delta: 'first durable output' }, + observedAt: LAST_CHUNK_AT + }) + vi.spyOn(Date, 'now').mockReturnValue(RESTARTED_AT - 1_000) + translator.flush() + await rig.host.flushStreamedEvents(SESSION) + translator.dispose() + const before = await rig.host.journalSnapshot(SESSION) + expect( + before.items.find((item) => JSON.stringify(item.body).includes('first durable output')) + ?.observedAt + ).toBe(HOST_TEST_NOW) + await expectInterruptedOutput() + }) + + it.each(['text_delta', 'thinking_delta'] as const)( + 'times delayed Claude %s checkpoints', + async (type) => { + const events = await beginTurn() + const translator = createClaudeJournalTranslator({ sink: events, schedule: () => () => {} }) + for (let chunk = 0; chunk <= 11; chunk += 1) { + translator.handle({ + type: 'message', + sessionId: SESSION, + observedAt: HOST_TEST_NOW + chunk * 500, + message: { + type: 'stream_event', + uuid: 'stream-frame', + session_id: 'claude-provider', + parent_tool_use_id: null, + event: { + type: 'content_block_delta', + index: 0, + delta: { type, text: `text chunk ${chunk} `, thinking: `thought chunk ${chunk} ` } + } + } + }) + } + vi.spyOn(Date, 'now').mockReturnValue(RESTARTED_AT - 1_000) + translator.flush() + await rig.host.flushStreamedEvents(SESSION) + translator.dispose() + const before = await rig.host.journalSnapshot(SESSION) + const output = before.items.find((item) => JSON.stringify(item.body).includes('chunk 11')) + expect(output).toBeDefined() + if (type === 'thinking_delta') { + expect(output?.observedAt).toBe(HOST_TEST_NOW) + } + await expectInterruptedOutput() + } + ) + + it('counts a synthetic provider fallback while excluding a later host notice', async () => { + const events = await beginTurn() + events.appendItem( + { provider: 'orca', clientMessageId: 'provider-fallback' }, + { kind: 'status', text: 'provider output' }, + { turnScope: TURN_SCOPE, providerObservedAt: LAST_CHUNK_AT } + ) + await rig.host.flushStreamedEvents(SESSION) + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal + if (!journal) { + throw new Error('journal absent') + } + await journal.appendItem( + { provider: 'orca', clientMessageId: 'host-notice' }, + { kind: 'status', text: 'host notice' }, + { fence: 1, turnScope: TURN_SCOPE, observedAt: RESTARTED_AT - 1_000 } + ) + await expectInterruptedOutput() + }) + + it('counts a pending approval with no assistant output before the crash', async () => { + const events = await beginTurn() + events.appendItem( + REPLY, + { + kind: 'approval', + title: 'Run?', + detail: null, + options: [], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }, + { turnScope: TURN_SCOPE, observedAt: LAST_CHUNK_AT } + ) + await rig.host.flushStreamedEvents(SESSION) + await restartAfterCrash() + await rig.host.restoreReadableSessions() + const snapshot = await rig.host.journalSnapshot(SESSION) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) + expect(snapshot.items.find((item) => item.body.kind === 'approval')?.body).toMatchObject({ + resolution: { state: 'cancelled' } + }) + const [timing] = selectStructuredAgentTurnTimings(snapshot.items).values() + expect(completedStructuredAgentTurnSeconds(timing)).toBe(5) + }) + + it('cannot record interruption from a surviving owner before exit proof', async () => { + const events = await beginTurn() + events.appendItem( + REPLY, + { kind: 'status', text: 'working' }, + { turnScope: TURN_SCOPE, observedAt: LAST_CHUNK_AT } + ) + await rig.host.flushStreamedEvents(SESSION) + await restartAfterCrash() + rig.host.deps.probeOwner = async () => ({ + outcome: 'identity-matched', + matchedOn: ['process-start-time'] + }) + const stopOwnerProcess = vi.fn() + rig.host.deps.stopOwnerProcess = stopOwnerProcess + await rig.host.reconcileRestartLeases() + const snapshot = await rig.host.journalSnapshot(SESSION) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn?.state).toBe('unverifiable') + expect(turn?.completedAt).toBeUndefined() + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(rig.store.getRecord(SESSION)?.lease.deathEvidence).toBeFalsy() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index 4c73941ee5dc..42268a8fb3ab 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -21,7 +21,10 @@ import type { StructuredAgentSessionSinkAdmission } from './structured-agent-session-event-sink' import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' -import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' +import { + providerObservedAppendOptions, + structuredAgentSessionJournalAppendOptions +} from './structured-agent-session-journal-append-options' /** What a transition step reads: rows by key, every row, and the turns they joined. */ export type StructuredAgentSessionTransitionJournal = Pick< @@ -84,8 +87,17 @@ export function createStructuredAgentSessionTransitionMembers( function transitionAppend( queue: StructuredAgentSessionSinkQueue ): (transition: StructuredAgentSessionTransition) => StructuredAgentSessionSinkAdmission { - return (transition) => - queue.submit({ + return (transition) => { + const providerObservedAt = Date.now() + const steps = transition.steps.map((step) => + step.kind === 'item' + ? { + ...step, + options: providerObservedAppendOptions({ providerObservedAt, ...step.options }) + } + : step + ) + return queue.submit({ bytes: transition.steps.reduce((total, step) => total + step.reservedBytes, 0) + (transition.publish ? 1 : 0), @@ -94,7 +106,7 @@ function transitionAppend( run: async (bound) => { const { journal, fence } = bound const wrote = await journal.appendSteps( - transition.steps.map((step): JournalStep => + steps.map((step): JournalStep => step.kind === 'item' ? { kind: 'item', @@ -116,6 +128,7 @@ function transitionAppend( batch: { settlementId: step.settlementId, fence, + providerObservedAt, resolve: () => step.resolve(journal) } } @@ -126,4 +139,5 @@ function transitionAppend( } } }) + } } From 32c65ea61ca74c16b41c9b6077591d7ac6a028d0 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 01:50:51 -0700 Subject: [PATCH 04/13] test: verify provider receipt in Claude ownership assertion --- .../claude-structured-journal-translation-subagents.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/claude/claude-structured-journal-translation-subagents.test.ts b/src/main/claude/claude-structured-journal-translation-subagents.test.ts index e1da571eb348..72c75c7925db 100644 --- a/src/main/claude/claude-structured-journal-translation-subagents.test.ts +++ b/src/main/claude/claude-structured-journal-translation-subagents.test.ts @@ -510,7 +510,8 @@ describe('claude journal translation — which agent produced a row', () => { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_bash', content: 'a.ts' }] } - } + }, + observedAt: 1_800_000_005_500 }) const row = items.find( @@ -518,6 +519,7 @@ describe('claude journal translation — which agent produced a row', () => { ) expect(row?.body).toMatchObject({ kind: 'tool-call', state: 'completed' }) expect(row?.options).toEqual({ + providerObservedAt: 1_800_000_005_500, turnScope: { kind: 'turn', turnItemId: expect.any(String) } }) }) From 28c23f05c524ccf4bcf3e5d6cba8cf96167e72ae Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:06:23 -0700 Subject: [PATCH 05/13] fix(native-chat): time interrupted turns from real provider output, not flushes Shared-timeline checkpoints stamped the coalescer flush or retry time as the provider's latest proof of life, so a turn whose provider died before a delayed flush settled with an inflated duration after a host restart. Carry each delta's receipt time through the coalescer and timeline lane. --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + src/main/jsonl-rpc/timeline-lane.test.ts | 28 ++ src/main/jsonl-rpc/timeline-lane.ts | 11 +- .../provider-timeline-assembler.ts | 31 +- .../provider-timeline-plan.ts | 10 +- .../provider-timeline-text-events.ts | 6 +- .../provider-timeline-text-streams.ts | 20 +- ...t-session-restart-timeline-receipt.test.ts | 270 ++++++++++++++++++ .../structured-agent-session-transition.ts | 5 +- 9 files changed, 358 insertions(+), 24 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 8a3e6e0468eb..fcff6b883ceb 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -117,6 +117,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts', diff --git a/src/main/jsonl-rpc/timeline-lane.test.ts b/src/main/jsonl-rpc/timeline-lane.test.ts index 8ef584d1ac35..aac7f400febe 100644 --- a/src/main/jsonl-rpc/timeline-lane.test.ts +++ b/src/main/jsonl-rpc/timeline-lane.test.ts @@ -15,6 +15,7 @@ afterEach(async () => { lane.dispose() } vi.useRealTimers() + vi.restoreAllMocks() await closeProviderTimelineRigs() }) @@ -51,6 +52,33 @@ async function rig() { } describe('JSON-lines timeline admission', () => { + it('retains receipts on item and settlement steps held before admission', async () => { + const test = await rig() + const cursor = test.journal.journal.cursor() + vi.spyOn(Date, 'now').mockReturnValue(5_500) + test.lane.apply([ + { type: 'turn.open', turn: 'run:1', at: 1_000 }, + { + type: 'item.open', + item: 'tool', + body: { kind: 'tool-call', name: 'read', input: {}, state: 'running' } + }, + { type: 'turn.end', at: 5_500, state: 'completed', outcome: 'success' } + ]) + vi.spyOn(Date, 'now').mockReturnValue(60_000) + test.unblock() + test.lane.retry() + await test.journal.rows() + const replay = test.journal.journal.readSince(cursor) + if (!replay.ok) { + throw new Error('journal replay reset') + } + expect(replay.rows.filter((row) => row.kind === 'item')).toHaveLength(2) + expect(replay.rows.some((row) => row.kind === 'lifecycle-batch')).toBe(true) + expect(replay.rows.every((row) => row.providerObservedAt === 5_500)).toBe(true) + expect(test.journal.journal.lastProviderActivityAt(1)).toBe(5_500) + }) + it('admits the final tail with the host settlement budget when ordinary writes are full', async () => { const journal = await openProviderTimelineRig({ agent: 'pi' }) const lane = new JsonlRpcTimelineLane({ diff --git a/src/main/jsonl-rpc/timeline-lane.ts b/src/main/jsonl-rpc/timeline-lane.ts index 36c759e42ded..dfadccc3b2a7 100644 --- a/src/main/jsonl-rpc/timeline-lane.ts +++ b/src/main/jsonl-rpc/timeline-lane.ts @@ -27,7 +27,11 @@ export type JsonlRpcTimelineLaneDeps = { export class JsonlRpcTimelineLane { private readonly assembler: ProviderTimelineAssembler private readonly identity: ReturnType - private readonly held: { event: ProviderTimelineEvent; bytes: number }[] = [] + private readonly held: { + event: ProviderTimelineEvent + bytes: number + providerObservedAt: number + }[] = [] private heldBytes = 0 private draining = false private ended = false @@ -66,6 +70,7 @@ export class JsonlRpcTimelineLane { } apply(events: readonly ProviderTimelineEvent[]): void { + const providerObservedAt = Date.now() if (this.ended) { return } @@ -75,7 +80,7 @@ export class JsonlRpcTimelineLane { this.fail('Agent timeline queue capacity exceeded') return } - this.held.push({ event, bytes }) + this.held.push({ event, bytes, providerObservedAt }) this.heldBytes += bytes this.retry() } @@ -91,7 +96,7 @@ export class JsonlRpcTimelineLane { try { while (this.held.length > 0 && !this.ended) { const entry = this.held[0] - const result = this.assembler.apply(entry.event) + const result = this.assembler.apply(entry.event, entry.providerObservedAt) if (!result.admission.accepted) { if (result.admission.reason !== 'backpressure') { this.fail(result.admission.reason) diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts index cee9087f240a..f61767a0e0a1 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts @@ -53,7 +53,7 @@ export type ProviderTimelineAssembler = { /** The host already wrote this command's running turn; the provider ends that same row. */ beginCommand(command: StructuredAgentSessionCommandRun): void forgetCommand(turnId: string): void - apply(event: ProviderTimelineEvent): ProviderTimelineApplyResult + apply(event: ProviderTimelineEvent, providerObservedAt?: number): ProviderTimelineApplyResult /** The turn id of the open turn, as its row and a client's Stop name it. */ readonly openTurnId: string | null /** Writes the text the coalescing window holds. */ @@ -103,11 +103,15 @@ export function createProviderTimelineAssembler( ...(deps.schedule ? { schedule: deps.schedule } : {}) }) - const textHost = (journal: ProviderTimelineTextHost['journal']): ProviderTimelineTextHost => ({ + const textHost = ( + journal: ProviderTimelineTextHost['journal'], + providerObservedAt: number + ): ProviderTimelineTextHost => ({ sink: deps.sink, state, streams, journal, + providerObservedAt, admits: (hold) => admits(hold, journal) }) @@ -116,7 +120,8 @@ export function createProviderTimelineAssembler( const applyDecided = ( event: ProviderTimelineDecidedEvent, - journal: ProviderTimelineTextHost['journal'] + journal: ProviderTimelineTextHost['journal'], + providerObservedAt: number ): ProviderTimelineApplyResult => { const serials = state.serials() const decision = decideProviderTimelineEvent( @@ -129,7 +134,7 @@ export function createProviderTimelineAssembler( if (decision.hold && !admits(decision.hold, journal)) { return { admission: PROVIDER_TIMELINE_OVER_BUDGET } } - const plan = new ProviderTimelinePlan() + const plan = new ProviderTimelinePlan(providerObservedAt) planProviderTimelineBarrier({ streams, state }, plan, event, decision) planProviderTimelineWrites(context, plan, decision, serials.next) plan.onAdmitted(() => { @@ -151,26 +156,30 @@ export function createProviderTimelineAssembler( * that stops its text and cancels its prompts; refused, the event that found it is refused with * it and its retry finds it again. */ const endSettledTurn = ( - journal: NonNullable + journal: NonNullable, + providerObservedAt: number ): ProviderTimelineApplyResult | null => { const open = state.open if (state.ended || !open || providerTimelineTurnRowState(journal, open.itemId) !== 'settled') { return null } - return applyDecided({ type: 'turn.settled', turn: open }, journal) + return applyDecided({ type: 'turn.settled', turn: open }, journal, providerObservedAt) } - const apply = (event: ProviderTimelineEvent): ProviderTimelineApplyResult => { + const apply = ( + event: ProviderTimelineEvent, + providerObservedAt = Date.now() + ): ProviderTimelineApplyResult => { const journal = deps.sink.journalItems() - const settled = journal ? endSettledTurn(journal) : null + const settled = journal ? endSettledTurn(journal, providerObservedAt) : null if (settled && !settled.admission.accepted) { return settled } switch (event.type) { case 'text.delta': - return applyProviderTimelineTextDelta(textHost(journal), event) + return applyProviderTimelineTextDelta(textHost(journal, providerObservedAt), event) case 'text.close': - return applyProviderTimelineTextClose(textHost(journal), event) + return applyProviderTimelineTextClose(textHost(journal, providerObservedAt), event) case 'activity': { const open = state.open if (state.ended || !open) { @@ -192,7 +201,7 @@ export function createProviderTimelineAssembler( case 'context.usage': case 'provider.frame': case 'session.ended': - return applyDecided(event, journal) + return applyDecided(event, journal, providerObservedAt) } } diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts index f9e9e0e33e38..4f70041ac2ef 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts @@ -28,13 +28,19 @@ export class ProviderTimelinePlan { private readonly admitted: (() => void)[] = [] private lifecycle = false + constructor(readonly providerObservedAt: number) {} + item(step: Omit, lifecycle = false): void { - this.steps.push({ kind: 'item', ...step }) + this.steps.push({ + kind: 'item', + ...step, + options: { providerObservedAt: this.providerObservedAt, ...step.options } + }) this.lifecycle ||= lifecycle } settlement(step: Omit): void { - this.steps.push({ kind: 'settlement', ...step }) + this.steps.push({ kind: 'settlement', providerObservedAt: this.providerObservedAt, ...step }) this.lifecycle = true } diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts index ec93367efd48..457a4ca56726 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts @@ -24,6 +24,7 @@ export type ProviderTimelineTextHost = { state: ProviderTimelineState streams: ProviderTimelineTextStreams journal: StructuredAgentSessionTransitionJournal | null + providerObservedAt: number admits(hold: ProviderTimelineHold): boolean } @@ -37,7 +38,7 @@ export function applyProviderTimelineTextDelta( if (state.ended) { return { admission: ADMITTED, dropped: 'session-ended' } } - const plan = new ProviderTimelinePlan() + const plan = new ProviderTimelinePlan(host.providerObservedAt) const key = streams.key(event.item, event.join) let stream = streams.get(key) if (stream && !streams.continues(stream, event.channel, event.producer)) { @@ -64,6 +65,7 @@ export function applyProviderTimelineTextDelta( channel: event.channel, producer: event.producer, state, + providerObservedAt: host.providerObservedAt, serial: serials.next() }) if (!host.admits({ key: stream.key, bytes: stream.bytes })) { @@ -105,7 +107,7 @@ export function applyProviderTimelineTextClose( dropped: streams.stoppedFor(key, event.join) ? 'turn-settled' : 'stream-unknown' } } - const plan = new ProviderTimelinePlan() + const plan = new ProviderTimelinePlan(host.providerObservedAt) streams.planFlush(plan, stream) streams.planClose(plan, stream, event.text) if (stream.named) { diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts index eda78997e6d2..366b5e76c5c2 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts @@ -60,6 +60,8 @@ export type ProviderTimelineStream = { named: boolean channel: ProviderTimelineTextChannel producer: AgentJournalProducerLinkage | undefined + startedAt: number + providerObservedAt: number /** Whether any text was written; a whitespace-only stream completes nothing. */ written: boolean bytes: number @@ -135,6 +137,7 @@ export class ProviderTimelineTextStreams { channel: ProviderTimelineTextChannel producer: AgentJournalProducerLinkage | undefined state: ProviderTimelineState + providerObservedAt: number serial: number }): ProviderTimelineStream { const { rows } = this.deps.context @@ -149,6 +152,8 @@ export class ProviderTimelineTextStreams { named, channel: input.channel, producer: input.producer, + startedAt: input.providerObservedAt, + providerObservedAt: input.providerObservedAt, written: false, bytes: providerTimelineEntryBytes({ key: 'id' in input.item ? input.item.id : input.item.stream, @@ -175,6 +180,7 @@ export class ProviderTimelineTextStreams { this.streams.set(stream.key, stream) this.byId.set(stream.id, stream) } + stream.providerObservedAt = Math.max(stream.providerObservedAt, plan.providerObservedAt) this.coalescer.append(stream.id, text) }) } @@ -242,7 +248,7 @@ export class ProviderTimelineTextStreams { } } else { const text = boundInlineText(finalText, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text - this.planText(plan, stream, text, true) + this.planText(plan, stream, text, true, plan.providerObservedAt) } this.planRelease(plan, (each) => each === stream) } @@ -264,7 +270,7 @@ export class ProviderTimelineTextStreams { if (!stream) { return true } - const plan = new ProviderTimelinePlan() + const plan = new ProviderTimelinePlan(stream.providerObservedAt) this.planText(plan, stream, text) const admission = plan.submit(this.deps.sink) // A failed or closed sink can never take it; the session's end owns what it leaves. @@ -275,7 +281,8 @@ export class ProviderTimelineTextStreams { plan: ProviderTimelinePlan, stream: ProviderTimelineStream, text: string, - providerFinal = false + providerFinal = false, + providerObservedAt = stream.providerObservedAt ): void { const empty = providerFinal ? text.length === 0 : text.trim().length === 0 if (!stream.written && empty) { @@ -286,7 +293,12 @@ export class ProviderTimelineTextStreams { plan.item({ reservedBytes: estimateStructuredAgentSessionItemBytes(stream.row.identity, body), resolve: (journal) => this.resolveText(stream, body, journal), - options: { ...stream.producer, turnScope: stream.scope } + options: { + ...stream.producer, + turnScope: stream.scope, + observedAt: stream.startedAt, + providerObservedAt + } }) plan.onAdmitted(() => { stream.written = true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts new file mode 100644 index 000000000000..93ced01c8dd9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts @@ -0,0 +1,270 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { JsonlRpcTimelineLane } from '../../jsonl-rpc/timeline-lane' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { createProviderTimelineAssembler } from '../agent-session-timeline/provider-timeline-assembler' +import { refusingSink } from '../agent-session-timeline/provider-timeline-assembler-test-support' +import { providerTimelineSink } from '../agent-session-timeline/provider-timeline-plan' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { HOST_TEST_NOW, hostTestAttachParams } from './structured-agent-session-host-test-data' +import { + createRestTestRig, + REST_TEST_CALLER, + restTestSend, + REST_TEST_SESSION as SESSION, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const LAST_OUTPUT_AT = HOST_TEST_NOW + 5_500 +const FLUSHED_AT = HOST_TEST_NOW + 60_000 +const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await rig.dispose() + closeTestJournalHostDatabases() +}) + +async function restartAfterCrash(): Promise { + const old = rig.host + old.stopDelivery() + const { lifetime, runtimeState, sessions } = old.collaboratorsForTests() + lifetime.idleSweep.dispose() + await runtimeState.stopLeaseRenewal() + runtimeState.currentEventSink(SESSION)?.close() + await sessions.get(SESSION)?.journal.close() + closeTestJournalHostDatabases() + rig.store = await openTestAgentSessionRecordStore(rig.root) + rig.host = new StructuredAgentSessionHost({ + ...old.deps, + store: rig.store, + journalDatabase: openTestJournalHostDatabase(rig.root), + now: () => RESTARTED_AT + }) +} + +async function providerSink() { + const attached = await rig.host.attach(REST_TEST_CALLER, hostTestAttachParams(null)) + if (!attached.ok) { + throw new Error('attach refused') + } + const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('stream', attached.fence)) + if (!sent.ok) { + throw new Error('send refused') + } + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) + const events = rig.adapter.acquire.mock.calls[0]?.[0].events + if (!events) { + throw new Error('provider sink missing') + } + const sink = providerTimelineSink(events) + if (!sink) { + throw new Error('timeline sink missing') + } + return sink +} + +async function expectInterruptedAt(at: number): Promise { + await restartAfterCrash() + await rig.host.restoreReadableSessions() + const snapshot = await rig.host.journalSnapshot(SESSION) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn).toMatchObject({ state: 'interrupted', completedAt: at }) +} + +describe('shared provider timeline receipt before coalescing', () => { + it.each([ + ['assistant', 'flush'], + ['reasoning', 'flush'], + ['assistant', 'window'], + ['reasoning', 'window'] + ] as const)( + 'ends a crashed %s stream at its last delta rather than a delayed %s', + async (channel, trigger) => { + const sink = await providerSink() + let receivedAt = HOST_TEST_NOW + vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) + let window = () => {} + const timeline = createProviderTimelineAssembler({ + sink, + sessionId: SESSION, + agent: 'opencode', + generation: 'review-generation', + namespace: 'review-acp', + schedule: (run) => { + window = run + return () => {} + } + }) + expect( + timeline.apply({ type: 'turn.open', turn: 'timeline-turn', at: HOST_TEST_NOW }) + ).toMatchObject({ admission: { accepted: true } }) + await rig.host.flushStreamedEvents(SESSION) + for (let chunk = 1; chunk <= 11; chunk += 1) { + receivedAt = HOST_TEST_NOW + chunk * 500 + expect( + timeline.apply({ + type: 'text.delta', + item: { stream: 'reply' }, + channel, + text: `chunk ${chunk} `, + join: { turn: 'timeline-turn' } + }) + ).toMatchObject({ admission: { accepted: true } }) + } + receivedAt = FLUSHED_AT + if (trigger === 'window') { + window() + } else { + timeline.flush() + } + await rig.host.flushStreamedEvents(SESSION) + timeline.dispose() + const before = await rig.host.journalSnapshot(SESSION) + const reply = before.items.find((item) => JSON.stringify(item.body).includes('chunk 11')) + expect(reply?.observedAt).toBe(HOST_TEST_NOW + 500) + expect(rig.store.getRecord(SESSION)?.lease.lastRenewedAt).toBe(HOST_TEST_NOW) + + await expectInterruptedAt(LAST_OUTPUT_AT) + } + ) + + it.each(['assistant', 'reasoning'] as const)( + 'retains held %s delta receipts across lane and coalescer retries', + async (channel) => { + const sink = await providerSink() + let receivedAt = HOST_TEST_NOW + vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) + let blocked = true + const lane = new JsonlRpcTimelineLane({ + sink: refusingSink(sink, () => blocked), + sessionId: SESSION, + agent: 'pi', + generation: 'receipt-generation', + namespace: 'receipt-pi', + pauseReading: vi.fn(), + resumeReading: vi.fn(), + onInputAccepted: vi.fn(), + onFailed: vi.fn() + }) + try { + lane.apply([{ type: 'turn.open', turn: 'held-turn', at: HOST_TEST_NOW }]) + for (let chunk = 1; chunk <= 11; chunk += 1) { + receivedAt = HOST_TEST_NOW + chunk * 500 + lane.apply([ + { + type: 'text.delta', + item: { stream: 'reply' }, + channel, + text: `held chunk ${chunk} `, + join: { turn: 'held-turn' } + } + ]) + } + receivedAt = FLUSHED_AT + blocked = false + lane.retry() + blocked = true + lane.flush() + await rig.host.flushStreamedEvents(SESSION) + const refused = await rig.host.journalSnapshot(SESSION) + expect(refused.items.some((item) => JSON.stringify(item.body).includes('held chunk'))).toBe( + false + ) + receivedAt += 60_000 + blocked = false + lane.flush() + await rig.host.flushStreamedEvents(SESSION) + const before = await rig.host.journalSnapshot(SESSION) + const reply = before.items.find((item) => + JSON.stringify(item.body).includes('held chunk 11') + ) + expect(reply?.observedAt).toBe(HOST_TEST_NOW + 500) + } finally { + lane.dispose() + } + await expectInterruptedAt(LAST_OUTPUT_AT) + } + ) + + it('keeps a held turn start separate from its receipt and later admission', async () => { + const sink = await providerSink() + vi.spyOn(Date, 'now').mockReturnValue(LAST_OUTPUT_AT) + let blocked = true + const lane = new JsonlRpcTimelineLane({ + sink: refusingSink(sink, () => blocked), + sessionId: SESSION, + agent: 'opencode', + generation: 'held-open-generation', + namespace: 'held-open-acp', + pauseReading: vi.fn(), + resumeReading: vi.fn(), + onInputAccepted: vi.fn(), + onFailed: vi.fn() + }) + try { + lane.apply([{ type: 'turn.open', turn: 'held-turn', at: HOST_TEST_NOW }]) + vi.spyOn(Date, 'now').mockReturnValue(FLUSHED_AT) + blocked = false + lane.retry() + await rig.host.flushStreamedEvents(SESSION) + } finally { + lane.dispose() + } + await expectInterruptedAt(LAST_OUTPUT_AT) + }) + + it('records a held final text snapshot at its receipt rather than the lane retry', async () => { + const sink = await providerSink() + let receivedAt = HOST_TEST_NOW + vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) + let blocked = false + const lane = new JsonlRpcTimelineLane({ + sink: refusingSink(sink, () => blocked), + sessionId: SESSION, + agent: 'pi', + generation: 'final-generation', + namespace: 'final-pi', + pauseReading: vi.fn(), + resumeReading: vi.fn(), + onInputAccepted: vi.fn(), + onFailed: vi.fn() + }) + try { + lane.apply([{ type: 'turn.open', turn: 'held-turn', at: HOST_TEST_NOW }]) + receivedAt += 500 + lane.apply([ + { + type: 'text.delta', + item: { stream: 'reply' }, + channel: 'assistant', + text: 'partial' + } + ]) + lane.flush() + await rig.host.flushStreamedEvents(SESSION) + receivedAt = LAST_OUTPUT_AT + blocked = true + lane.apply([{ type: 'text.close', item: { stream: 'reply' }, text: 'final reply' }]) + receivedAt = FLUSHED_AT + blocked = false + lane.retry() + await rig.host.flushStreamedEvents(SESSION) + const before = await rig.host.journalSnapshot(SESSION) + const reply = before.items.find((item) => JSON.stringify(item.body).includes('final reply')) + expect(reply?.observedAt).toBe(HOST_TEST_NOW + 500) + } finally { + lane.dispose() + } + await expectInterruptedAt(LAST_OUTPUT_AT) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index 42268a8fb3ab..d2c3dc40b220 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -48,6 +48,7 @@ export type StructuredAgentSessionTransitionStep = kind: 'settlement' /** Unique per settlement: the journal applies one id once. */ settlementId: string + providerObservedAt?: number /** Paces the queue only; the mutations are the journal's to choose. */ reservedBytes: number /** Read at execution; none writes nothing. */ @@ -93,7 +94,7 @@ function transitionAppend( step.kind === 'item' ? { ...step, - options: providerObservedAppendOptions({ providerObservedAt, ...step.options }) + options: providerObservedAppendOptions(step.options) } : step ) @@ -128,7 +129,7 @@ function transitionAppend( batch: { settlementId: step.settlementId, fence, - providerObservedAt, + providerObservedAt: step.providerObservedAt ?? providerObservedAt, resolve: () => step.resolve(journal) } } From 395c63b4bdd69c1e1f9c5852611ba61e93529da8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:53:27 -0700 Subject: [PATCH 06/13] fix(native-chat): derive interrupted-turn duration from saved output A reply cut off by a server restart could show "Worked for 0s" because recovery ended the turn at the last heartbeat or the reply's first timestamp. End it at the latest time this owner's output was saved to chat history instead, derived in the existing journal fold from the row save time. Drops the per-frame receipt plumbing through every provider and buffer. --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 - src/main/claude/claude-message-journaling.ts | 16 +- .../claude-streamed-text-checkpoints.test.ts | 18 +- .../claude-streamed-text-checkpoints.ts | 17 +- src/main/claude/claude-streamed-thinking.ts | 2 +- ...ured-journal-translation-subagents.test.ts | 4 +- .../claude-structured-journal-translation.ts | 2 +- .../codex-structured-item-stream-contracts.ts | 6 +- .../codex/codex-structured-item-streams.ts | 46 +-- .../codex/codex-structured-journal-items.ts | 7 +- .../codex/codex-structured-journal-sink.ts | 22 -- .../codex-structured-journal-translation.ts | 7 +- src/main/jsonl-rpc/timeline-lane.test.ts | 28 -- src/main/jsonl-rpc/timeline-lane.ts | 11 +- .../agent-session-journal/journal-item-row.ts | 49 ---- .../agent-session-journal/journal-open.ts | 6 +- .../journal-provider-activity.ts | 15 +- .../agent-session-journal/journal-reducer.ts | 12 +- .../agent-session-journal/journal-row-base.ts | 11 - .../journal-row-builders.ts | 63 ++-- .../journal-row-schema.test.ts | 42 --- .../journal-row-schema.ts | 2 - .../journal-row-table.ts | 6 +- .../journal-row-unusable-annotations.ts | 8 - .../journal-row-writer.test.ts | 18 ++ .../journal-row-writer.ts | 13 +- .../journal-store-collaborators.ts | 2 +- .../journal-store-contracts.ts | 2 - .../agent-session-journal/journal-store.ts | 6 +- .../provider-timeline-assembler.ts | 31 +- .../provider-timeline-plan.ts | 10 +- .../provider-timeline-text-events.ts | 6 +- .../provider-timeline-text-streams.ts | 20 +- ...tructured-agent-session-event-sink.test.ts | 35 +-- .../structured-agent-session-event-sink.ts | 33 ++- .../structured-agent-session-item-append.ts | 27 -- ...ed-agent-session-journal-append-options.ts | 18 +- ...tructured-agent-session-resolved-append.ts | 13 +- ...nt-session-restart-provider-output.test.ts | 44 ++- ...nt-session-restart-stream-duration.test.ts | 11 +- ...t-session-restart-timeline-receipt.test.ts | 270 ------------------ ...ctured-agent-session-stale-turn-verdict.ts | 6 +- .../structured-agent-session-transition.ts | 23 +- 43 files changed, 246 insertions(+), 743 deletions(-) delete mode 100644 src/main/native-chat/agent-session-journal/journal-item-row.ts delete mode 100644 src/main/native-chat/agent-session-journal/journal-row-base.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index fcff6b883ceb..8a3e6e0468eb 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -117,7 +117,6 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts', - 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts', diff --git a/src/main/claude/claude-message-journaling.ts b/src/main/claude/claude-message-journaling.ts index 4ff785e368e5..9d2e00a2e67f 100644 --- a/src/main/claude/claude-message-journaling.ts +++ b/src/main/claude/claude-message-journaling.ts @@ -108,10 +108,7 @@ export function journalClaudeMessage( // output; a reader that scans back to the turn record and stops would // otherwise look straight past the row that opened it. ctx.turn.ensureOpen(message, source, observedAt) - ctx.sink.appendItem(identity, body, { - ...stamp(identity, body), - providerObservedAt: observedAt - }) + ctx.sink.appendItem(identity, body, stamp(identity, body)) changed = true } for (const tool of claudeToolUses(outputEnvelope)) { @@ -128,10 +125,7 @@ export function journalClaudeMessage( } const toolIdentity = claudeToolIdentity(envelope.sessionId, tool.id) const toolBody = claudeToolBody({ tool }) - ctx.sink.appendItem(toolIdentity, toolBody, { - ...stamp(toolIdentity, toolBody), - providerObservedAt: observedAt - }) + ctx.sink.appendItem(toolIdentity, toolBody, stamp(toolIdentity, toolBody)) changed = true } for (const result of results) { @@ -142,10 +136,7 @@ export function journalClaudeMessage( } const resultIdentity = claudeToolIdentity(envelope.sessionId, result.toolUseId) const resultBody = claudeToolBody({ tool, result }) - ctx.sink.appendItem(resultIdentity, resultBody, { - ...stamp(resultIdentity, resultBody), - providerObservedAt: observedAt - }) + ctx.sink.appendItem(resultIdentity, resultBody, stamp(resultIdentity, resultBody)) ctx.subagents.observeToolResult(result.toolUseId, result.failed) if ( results.length === 1 && @@ -163,7 +154,6 @@ export function journalClaudeMessage( // The write that ends the row: shed under pressure, the row would read open for good. ctx.sink.appendItem(thinking.identity, thinking.body, { ...stamp(thinking.identity, thinking.body), - providerObservedAt: observedAt, ...(thinking.startedAt === undefined ? {} : { observedAt: thinking.startedAt }), lifecycle: true }) diff --git a/src/main/claude/claude-streamed-text-checkpoints.test.ts b/src/main/claude/claude-streamed-text-checkpoints.test.ts index ce4df1f7e9c4..89f825bc5605 100644 --- a/src/main/claude/claude-streamed-text-checkpoints.test.ts +++ b/src/main/claude/claude-streamed-text-checkpoints.test.ts @@ -142,7 +142,7 @@ describe('claude streamed text checkpoints', () => { runWindow() expect(rows).toEqual([{ uuid: 'block-1', text: 'hello' }]) - expect(stamps).toEqual([{ ...CHILD_LINKAGE, providerObservedAt: expect.any(Number) }]) + expect(stamps).toEqual([CHILD_LINKAGE]) }) it("writes no linkage keys for a block the session's own agent streamed", () => { @@ -151,7 +151,7 @@ describe('claude streamed text checkpoints', () => { store.append(identityOf('block-1'), 'hello') runWindow() - expect(stamps).toEqual([{ providerObservedAt: expect.any(Number) }]) + expect(stamps).toEqual([{}]) }) it('writes a checkpoint at once while the producing agent is provisional', () => { @@ -176,7 +176,7 @@ describe('claude streamed text checkpoints', () => { store.reattribute() expect(rows.at(-1)).toEqual({ uuid: 'block-1', text: 'partial' }) - expect(stamps.at(-1)).toEqual({ ...CHILD_LINKAGE, providerObservedAt: expect.any(Number) }) + expect(stamps.at(-1)).toEqual(CHILD_LINKAGE) }) it('writes a held block under the raw reference when no announcement comes', () => { @@ -190,12 +190,7 @@ describe('claude streamed text checkpoints', () => { expect(rows).toEqual([{ uuid: 'block-1', text: 'never announced' }]) expect(stamps).toEqual([ - { - agentId: 'toolu_1', - providerParentRef: 'toolu_1', - producerKind: 'agent', - providerObservedAt: expect.any(Number) - } + { agentId: 'toolu_1', providerParentRef: 'toolu_1', producerKind: 'agent' } ]) // Nothing ever names it, so re-attribution has nothing better to say and @@ -218,10 +213,7 @@ describe('claude streamed text checkpoints', () => { store.append(identityOf('block-1'), 'first and more', 'toolu_1') store.flush() - expect(stamps).toEqual([ - { ...CHILD_LINKAGE, providerObservedAt: expect.any(Number) }, - { ...CHILD_LINKAGE, agentId: 'task-2', providerObservedAt: expect.any(Number) } - ]) + expect(stamps).toEqual([CHILD_LINKAGE, { ...CHILD_LINKAGE, agentId: 'task-2' }]) }) it('stops persisting once disposed', () => { diff --git a/src/main/claude/claude-streamed-text-checkpoints.ts b/src/main/claude/claude-streamed-text-checkpoints.ts index ee9d113a20f9..cf8d3c40be5b 100644 --- a/src/main/claude/claude-streamed-text-checkpoints.ts +++ b/src/main/claude/claude-streamed-text-checkpoints.ts @@ -31,8 +31,7 @@ export type ClaudeStreamedTextCheckpoints = { append: ( identity: AgentJournalItemIdentity, text: string, - parentToolUseId?: string | null, - providerObservedAt?: number + parentToolUseId?: string | null ) => void /** Write every block whose row is behind the text received for it. */ flush: () => void @@ -91,19 +90,16 @@ export function createClaudeStreamedTextCheckpoints( const writtenLinkage = new Map() const latestText = new Map() const checkpointLengths = new Map() - const observations = new Map() /** Resolved FRESH on every checkpoint. A provisional producer is stamped with * the handle it has rather than holding the prose back: the next checkpoint, * or `reattribute` once the announcement lands, revises the same row. */ const producerOptions = (key: string): StructuredAgentSessionAppendOptions => { const scope = scopes.get(key) ?? null - return { - ...(scope === null - ? {} - : agentJournalLinkageFields(deps.producer.settledLinkageFor(scope).linkage)), - providerObservedAt: observations.get(key) + if (scope === null) { + return {} } + return agentJournalLinkageFields(deps.producer.settledLinkageFor(scope).linkage) } const persist = (key: string, text: string, force: boolean): void => { @@ -136,15 +132,13 @@ export function createClaudeStreamedTextCheckpoints( writtenLinkage.delete(key) latestText.delete(key) checkpointLengths.delete(key) - observations.delete(key) } return { - append: (identity, text, parentToolUseId = null, providerObservedAt = Date.now()) => { + append: (identity, text, parentToolUseId = null) => { const key = agentJournalItemKey(identity) identities.set(key, identity) scopes.set(key, parentToolUseId) - observations.set(key, providerObservedAt) coalescer.append(key, text) }, flush: () => { @@ -202,7 +196,6 @@ export function createClaudeStreamedTextCheckpoints( writtenLinkage.clear() latestText.clear() checkpointLengths.clear() - observations.clear() } } } diff --git a/src/main/claude/claude-streamed-thinking.ts b/src/main/claude/claude-streamed-thinking.ts index 431cca283c4b..1edd109adebf 100644 --- a/src/main/claude/claude-streamed-thinking.ts +++ b/src/main/claude/claude-streamed-thinking.ts @@ -109,7 +109,7 @@ export function createClaudeStreamedThinking(deps: { startedAt: delta.startedAt }) } - checkpoints.append(identity, delta.text, delta.parentToolUseId, observedAt) + checkpoints.append(identity, delta.text, delta.parentToolUseId) return true }, /** The row a final frame's thinking lands on — its streamed block's, else its own — closed. diff --git a/src/main/claude/claude-structured-journal-translation-subagents.test.ts b/src/main/claude/claude-structured-journal-translation-subagents.test.ts index 72c75c7925db..e1da571eb348 100644 --- a/src/main/claude/claude-structured-journal-translation-subagents.test.ts +++ b/src/main/claude/claude-structured-journal-translation-subagents.test.ts @@ -510,8 +510,7 @@ describe('claude journal translation — which agent produced a row', () => { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_bash', content: 'a.ts' }] } - }, - observedAt: 1_800_000_005_500 + } }) const row = items.find( @@ -519,7 +518,6 @@ describe('claude journal translation — which agent produced a row', () => { ) expect(row?.body).toMatchObject({ kind: 'tool-call', state: 'completed' }) expect(row?.options).toEqual({ - providerObservedAt: 1_800_000_005_500, turnScope: { kind: 'turn', turnItemId: expect.any(String) } }) }) diff --git a/src/main/claude/claude-structured-journal-translation.ts b/src/main/claude/claude-structured-journal-translation.ts index 6b5dc5bef691..e2932e0d3468 100644 --- a/src/main/claude/claude-structured-journal-translation.ts +++ b/src/main/claude/claude-structured-journal-translation.ts @@ -168,7 +168,7 @@ export function createClaudeJournalTranslator( delta || thinking ? claudeStreamTurnSource(message) : claudeStreamTurnStartSource(message) turn.ensureOpen(message, source, observedAt) if (delta) { - streamedText.append(delta.identity, delta.text, delta.parentToolUseId, observedAt) + streamedText.append(delta.identity, delta.text, delta.parentToolUseId) } return delta !== null || thinking } diff --git a/src/main/codex/codex-structured-item-stream-contracts.ts b/src/main/codex/codex-structured-item-stream-contracts.ts index 6b64b2bcc892..38b9a9d3038a 100644 --- a/src/main/codex/codex-structured-item-stream-contracts.ts +++ b/src/main/codex/codex-structured-item-stream-contracts.ts @@ -19,7 +19,6 @@ export type CodexItemStreamDeps = { maxTotalRetainedBytes?: number maxMetadataBytes?: number schedule?: AgentSessionDeltaCoalescerDeps['schedule'] - now?: () => number } export type CodexItemStreamState = { @@ -27,13 +26,11 @@ export type CodexItemStreamState = { item: CodexThreadItem /** Host clock when the item started, for a row its stream writes first. */ startedAt?: number - providerObservedAt?: number } export type CodexPendingItemPatch = { identity: AgentJournalItemIdentity body: NonNullable['body']> - providerObservedAt: number } export type CodexStructuredItemStreamAdmission = @@ -58,8 +55,7 @@ export type CodexStructuredItemStreams = { handle: ( threadId: string, method: string, - params: unknown, - observedAt?: number + params: unknown ) => CodexStructuredItemStreamHandleResult forget: (threadId: string, itemId: string) => void flush: () => boolean diff --git a/src/main/codex/codex-structured-item-streams.ts b/src/main/codex/codex-structured-item-streams.ts index ae47ba361e23..c783458ca298 100644 --- a/src/main/codex/codex-structured-item-streams.ts +++ b/src/main/codex/codex-structured-item-streams.ts @@ -4,11 +4,9 @@ import { } from '../../shared/agent-session-journal-types' import { createAgentSessionDeltaCoalescer } from '../native-chat/agent-session-wire/agent-session-delta-coalescer' import { CodexItemStreamRetention } from './codex-item-stream-retention' -import { - appendCodexItemAndPublish, - appendCodexStreamCheckpoint -} from './codex-structured-journal-sink' -import { codexJournalItem } from './codex-structured-item-translation' +import { appendCodexItemAndPublish } from './codex-structured-journal-sink' +import { codexJournalItem, codexStreamingJournalItem } from './codex-structured-item-translation' +import { withJournalReasoningLifecycle } from '../native-chat/agent-session-journal/journal-reasoning-row' import { codexStructuredItemKey, MAX_CODEX_ITEM_STREAM_PENDING_PATCHES, @@ -111,6 +109,19 @@ export function createCodexStructuredItemStreams( } } + const append = (key: string, state: CodexItemStreamState, text: string): boolean => { + const translated = codexStreamingJournalItem(state.item, text) + if (!translated.body) { + return true + } + // A stream only ever carries an item that has not completed yet. + const body = withJournalReasoningLifecycle(translated.body, { state: 'running' }) + return appendCodexItemAndPublish(deps.sink, state.identity, body, { + ...attributionOf(key), + ...(state.startedAt === undefined ? {} : { observedAt: state.startedAt }) + }).accepted + } + const shouldPersist = (key: string, textLength: number): boolean => { const checkpointLength = checkpointLengths.get(key) ?? 0 const nextLength = Math.max(checkpointLength + 32, Math.ceil(checkpointLength * 1.125)) @@ -122,7 +133,7 @@ export function createCodexStructuredItemStreams( return true } const state = states.get(key) - if (state && appendCodexStreamCheckpoint(deps.sink, attributionOf(key), state, text)) { + if (state && append(key, state, text)) { checkpointLengths.set(key, text.length) pendingCheckpoints.delete(key) return true @@ -185,10 +196,12 @@ export function createCodexStructuredItemStreams( if (!pending) { return { accepted: true } } - const admission = appendCodexItemAndPublish(deps.sink, pending.identity, pending.body, { - ...attributionOf(key), - providerObservedAt: pending.providerObservedAt - }) + const admission = appendCodexItemAndPublish( + deps.sink, + pending.identity, + pending.body, + attributionOf(key) + ) if (!admission.accepted) { return admission } @@ -208,19 +221,14 @@ export function createCodexStructuredItemStreams( ), track: (threadId, turnId, item, identity, startedAt) => { const key = codexStructuredItemKey(threadId, item.id) - if ( - !states.retain(key, { - ...codexItemStreamState(item, identity, startedAt), - providerObservedAt: startedAt ?? Date.now() - }) - ) { + if (!states.retain(key, codexItemStreamState(item, identity, startedAt))) { return false } producers.set(key, { threadId, turnId }) trimStates() return true }, - handle: (threadId, method, params, observedAt) => { + handle: (threadId, method, params) => { const paramsRecord = readCodexItemStreamRecord(params) const itemId = readCodexItemStreamString(paramsRecord, 'itemId') if (method === PATCH_UPDATED_METHOD) { @@ -243,8 +251,7 @@ export function createCodexStructuredItemStreams( if (translated.body) { const nextPending: CodexPendingItemPatch = { identity: state.identity, - body: translated.body, - providerObservedAt: observedAt ?? deps.now?.() ?? Date.now() + body: translated.body } const previous = pendingPatches.get(key) const previousBytes = previous ? pendingPatchBytes(previous) : 0 @@ -284,7 +291,6 @@ export function createCodexStructuredItemStreams( const delta = method === REASONING_PART_METHOD ? '\n' : paramsRecord.delta if (typeof delta === 'string') { const key = codexStructuredItemKey(threadId, state.item.id) - state.providerObservedAt = observedAt ?? deps.now?.() ?? Date.now() pendingCheckpoints.add(key) const accepted = coalescer.append(key, delta) if (!accepted) { diff --git a/src/main/codex/codex-structured-journal-items.ts b/src/main/codex/codex-structured-journal-items.ts index 3a5a9066e53d..17a002225e7c 100644 --- a/src/main/codex/codex-structured-journal-items.ts +++ b/src/main/codex/codex-structured-journal-items.ts @@ -64,7 +64,6 @@ export class CodexJournalItems { coalesceMs: deps.coalesceMs, maxRetainedBytes: deps.maxRetainedBytes, schedule: deps.schedule, - now: deps.now, maxMetadataBytes: deps.maxMetadataBytes, turnIdFor: (threadId, params) => readCodexTurnId(params) ?? this.activeTurn(threadId), identityFor: (threadId, turnId, item) => this.identityFor(threadId, turnId, item), @@ -152,7 +151,6 @@ export class CodexJournalItems { const startedAt = event.method === 'item/completed' ? active?.startedAt : receivedAt const admission = this.appendTranslated(event.method, identity, translated, { ...this.deps.attributionFor(event.threadId, turnId), - providerObservedAt: receivedAt, ...(startedAt === undefined ? {} : { observedAt: startedAt }) }) if (!admission.accepted) { @@ -182,10 +180,7 @@ export class CodexJournalItems { method: string, identity: AgentJournalItemIdentity, translated: ReturnType, - attribution: AgentJournalRowAttribution & { - observedAt?: number - providerObservedAt?: number - } + attribution: AgentJournalRowAttribution & { observedAt?: number } ): CodexJournalTranslationAdmission { if (!translated.body) { return CODEX_JOURNAL_ADMITTED diff --git a/src/main/codex/codex-structured-journal-sink.ts b/src/main/codex/codex-structured-journal-sink.ts index ae573a903c12..dc4778ea3501 100644 --- a/src/main/codex/codex-structured-journal-sink.ts +++ b/src/main/codex/codex-structured-journal-sink.ts @@ -1,6 +1,3 @@ -import type { CodexItemStreamState } from './codex-structured-item-stream-contracts' -import { codexStreamingJournalItem } from './codex-structured-item-translation' -import { withJournalReasoningLifecycle } from '../native-chat/agent-session-journal/journal-reasoning-row' import type { AgentJournalItemBody, AgentJournalItemIdentity, @@ -167,22 +164,3 @@ export function admitCodexLifecycleItems( } return publishCodexLifecycle(sink) } - -export function appendCodexStreamCheckpoint( - sink: StructuredAgentSessionEventSink, - attribution: AgentJournalRowAttribution, - state: CodexItemStreamState, - text: string -): boolean { - const translated = codexStreamingJournalItem(state.item, text) - if (!translated.body) { - return true - } - // A stream only ever carries an item that has not completed yet. - const body = withJournalReasoningLifecycle(translated.body, { state: 'running' }) - return appendCodexItemAndPublish(sink, state.identity, body, { - ...attribution, - providerObservedAt: state.providerObservedAt, - ...(state.startedAt === undefined ? {} : { observedAt: state.startedAt }) - }).accepted -} diff --git a/src/main/codex/codex-structured-journal-translation.ts b/src/main/codex/codex-structured-journal-translation.ts index 1cc7088d4b2b..19fee37a5368 100644 --- a/src/main/codex/codex-structured-journal-translation.ts +++ b/src/main/codex/codex-structured-journal-translation.ts @@ -188,12 +188,7 @@ export function createCodexJournalTranslator( return CODEX_JOURNAL_ADMITTED } if (event.type === 'notification') { - const streamResult = items.streams.handle( - event.threadId, - event.method, - event.params, - event.observedAt - ) + const streamResult = items.streams.handle(event.threadId, event.method, event.params) if (streamResult.handled) { return publishActivity(event, streamResult.admission) } diff --git a/src/main/jsonl-rpc/timeline-lane.test.ts b/src/main/jsonl-rpc/timeline-lane.test.ts index aac7f400febe..8ef584d1ac35 100644 --- a/src/main/jsonl-rpc/timeline-lane.test.ts +++ b/src/main/jsonl-rpc/timeline-lane.test.ts @@ -15,7 +15,6 @@ afterEach(async () => { lane.dispose() } vi.useRealTimers() - vi.restoreAllMocks() await closeProviderTimelineRigs() }) @@ -52,33 +51,6 @@ async function rig() { } describe('JSON-lines timeline admission', () => { - it('retains receipts on item and settlement steps held before admission', async () => { - const test = await rig() - const cursor = test.journal.journal.cursor() - vi.spyOn(Date, 'now').mockReturnValue(5_500) - test.lane.apply([ - { type: 'turn.open', turn: 'run:1', at: 1_000 }, - { - type: 'item.open', - item: 'tool', - body: { kind: 'tool-call', name: 'read', input: {}, state: 'running' } - }, - { type: 'turn.end', at: 5_500, state: 'completed', outcome: 'success' } - ]) - vi.spyOn(Date, 'now').mockReturnValue(60_000) - test.unblock() - test.lane.retry() - await test.journal.rows() - const replay = test.journal.journal.readSince(cursor) - if (!replay.ok) { - throw new Error('journal replay reset') - } - expect(replay.rows.filter((row) => row.kind === 'item')).toHaveLength(2) - expect(replay.rows.some((row) => row.kind === 'lifecycle-batch')).toBe(true) - expect(replay.rows.every((row) => row.providerObservedAt === 5_500)).toBe(true) - expect(test.journal.journal.lastProviderActivityAt(1)).toBe(5_500) - }) - it('admits the final tail with the host settlement budget when ordinary writes are full', async () => { const journal = await openProviderTimelineRig({ agent: 'pi' }) const lane = new JsonlRpcTimelineLane({ diff --git a/src/main/jsonl-rpc/timeline-lane.ts b/src/main/jsonl-rpc/timeline-lane.ts index dfadccc3b2a7..36c759e42ded 100644 --- a/src/main/jsonl-rpc/timeline-lane.ts +++ b/src/main/jsonl-rpc/timeline-lane.ts @@ -27,11 +27,7 @@ export type JsonlRpcTimelineLaneDeps = { export class JsonlRpcTimelineLane { private readonly assembler: ProviderTimelineAssembler private readonly identity: ReturnType - private readonly held: { - event: ProviderTimelineEvent - bytes: number - providerObservedAt: number - }[] = [] + private readonly held: { event: ProviderTimelineEvent; bytes: number }[] = [] private heldBytes = 0 private draining = false private ended = false @@ -70,7 +66,6 @@ export class JsonlRpcTimelineLane { } apply(events: readonly ProviderTimelineEvent[]): void { - const providerObservedAt = Date.now() if (this.ended) { return } @@ -80,7 +75,7 @@ export class JsonlRpcTimelineLane { this.fail('Agent timeline queue capacity exceeded') return } - this.held.push({ event, bytes, providerObservedAt }) + this.held.push({ event, bytes }) this.heldBytes += bytes this.retry() } @@ -96,7 +91,7 @@ export class JsonlRpcTimelineLane { try { while (this.held.length > 0 && !this.ended) { const entry = this.held[0] - const result = this.assembler.apply(entry.event, entry.providerObservedAt) + const result = this.assembler.apply(entry.event) if (!result.admission.accepted) { if (result.admission.reason !== 'backpressure') { this.fail(result.admission.reason) diff --git a/src/main/native-chat/agent-session-journal/journal-item-row.ts b/src/main/native-chat/agent-session-journal/journal-item-row.ts deleted file mode 100644 index d673c0dd84d0..000000000000 --- a/src/main/native-chat/agent-session-journal/journal-item-row.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { - AgentJournalItemBody, - AgentJournalItemIdentity, - AgentJournalProducerLinkage, - AgentJournalTurnScope -} from '../../../shared/agent-session-journal-types' -import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' -import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' -import type { JournalReducerState } from './journal-reducer' -import type { JournalItemRow } from './journal-row-schema' -import { journalRowBase } from './journal-row-base' -import { turnEndAfterStop } from './journal-stop-turn-end' - -export function buildJournalItemRow(input: { - state: JournalReducerState - identity: AgentJournalItemIdentity - body: AgentJournalItemBody - seq: number - fence: number - ts: number - recovered?: true - providerObservedAt?: number - linkage?: AgentJournalProducerLinkage - turnScope: AgentJournalTurnScope -}): JournalItemRow { - const itemId = agentJournalItemKey(input.identity) - const resolved = input.state.aliases.get(itemId) ?? itemId - // A tombstoned row keeps its revision in `tombstones`, and the reducer drops - // any item at or below it — so a re-add has to outrank the tombstone too. - const revision = - Math.max( - input.state.items.get(resolved)?.revision ?? 0, - input.state.tombstones.get(resolved) ?? 0 - ) + 1 - const body = turnEndAfterStop(input.state, resolved, input.body) - return { - kind: 'item', - itemId, - revision, - body, - ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), - ...(input.recovered ? { recovered: input.recovered } : {}), - ...(input.providerObservedAt === undefined - ? {} - : { providerObservedAt: input.providerObservedAt }), - turnScope: input.turnScope, - ...agentJournalLinkageFields(input.linkage) - } -} diff --git a/src/main/native-chat/agent-session-journal/journal-open.ts b/src/main/native-chat/agent-session-journal/journal-open.ts index 2c066601ea7a..577efeccedd1 100644 --- a/src/main/native-chat/agent-session-journal/journal-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-open.ts @@ -57,7 +57,7 @@ export function replayJournal(db: Database.Database, sessionId: string): Journal /** The same fold, fed a row at a time, for a caller that yields between batches of rows. */ export function startJournalRowFold(input: { sessionId: string; epoch: string }): { /** False once the fold has stopped: the rest of the rows are not read. */ - add: (entry: { seq: number; rowJson: string }) => boolean + add: (entry: { seq: number; rowJson: string; ts?: number }) => boolean finish: () => JournalLoad } { const state = createJournalReducerState(input.sessionId, input.epoch) @@ -67,7 +67,7 @@ export function startJournalRowFold(input: { sessionId: string; epoch: string }) let newer: { sequence: number } | null = null let empty = true - const add = (entry: { seq: number; rowJson: string }): boolean => { + const add = (entry: { seq: number; rowJson: string; ts?: number }): boolean => { empty = false const parsed = parseJournalRow(entry.rowJson) if (!parsed.ok && parsed.unreadable) { @@ -99,7 +99,7 @@ export function startJournalRowFold(input: { sessionId: string; epoch: string }) return true } anchored = true - applyJournalRow(state, row) + applyJournalRow(state, row, entry.ts) return true } const finish = (): JournalLoad => { diff --git a/src/main/native-chat/agent-session-journal/journal-provider-activity.ts b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts index 260fb6397692..09a965b624ef 100644 --- a/src/main/native-chat/agent-session-journal/journal-provider-activity.ts +++ b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts @@ -4,23 +4,30 @@ import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { JournalReducerState } from './journal-reducer' import type { JournalRow } from './journal-row-schema' -/** Provider revisions prove life after a renewal; client actions and recovery writes do not. */ +/** Saved output extends a turn; client actions and recovery writes do not. */ export function observeJournalProviderActivity( state: Pick, row: JournalRow, itemId: string, - body: AgentJournalItemBody + body: AgentJournalItemBody, + savedAt: number ): void { if (row.recovered || !isProviderActivity(body)) { return } const identity = parseAgentJournalItemKey(itemId) - if (!identity || (identity.provider === 'orca' && row.providerObservedAt === undefined)) { + // Codex detail rows use the Orca namespace; fallback rows already name their provider. + if ( + !identity || + (identity.provider === 'orca' && + !identity.clientMessageId.startsWith('codex-item:') && + !(body.kind === 'status' && body.providerFrame !== undefined)) + ) { return } state.providerActivityAt.set( row.fence, - Math.max(state.providerActivityAt.get(row.fence) ?? 0, row.providerObservedAt ?? row.ts) + Math.max(state.providerActivityAt.get(row.fence) ?? 0, savedAt) ) } diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index d32ce339e8a2..eec6fdfa5892 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -51,7 +51,7 @@ export type JournalReducerState = { items: Map /** Fence of the writer that created each item: the generation a running turn belongs to. */ itemFences: Map - /** Latest provider observation per owner, rebuilt from rows including streaming revisions. */ + /** Latest saved output per owner, rebuilt during the existing row fold. */ providerActivityAt: Map /** Revision of a removed item, so a late lower revision cannot resurrect it. */ tombstones: Map @@ -92,7 +92,11 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou } } -export function applyJournalRow(state: JournalReducerState, row: JournalRow): void { +export function applyJournalRow( + state: JournalReducerState, + row: JournalRow, + savedAt = row.ts +): void { state.lastSequence = Math.max(state.lastSequence, row.seq) state.highestFence = Math.max(state.highestFence, row.fence) if (row.kind === 'epoch') { @@ -104,7 +108,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo return } const itemId = resolveJournalItemId(state, row.itemId, row.body) - observeJournalProviderActivity(state, row, row.itemId, row.body) + observeJournalProviderActivity(state, row, row.itemId, row.body, savedAt) acceptSubmissionFromProviderItem(state, row.itemId, itemId, row) upsertJournalItem( state, @@ -133,7 +137,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo continue } const { revision, body } = mutation - observeJournalProviderActivity(state, row, mutation.itemId, body) + observeJournalProviderActivity(state, row, mutation.itemId, body, savedAt) const itemId = resolveJournalItemId(state, mutation.itemId, body) acceptSubmissionFromProviderItem(state, mutation.itemId, itemId, row) const producer = journalBatchMutationProducer(row, mutation) diff --git a/src/main/native-chat/agent-session-journal/journal-row-base.ts b/src/main/native-chat/agent-session-journal/journal-row-base.ts deleted file mode 100644 index 2eb00979c936..000000000000 --- a/src/main/native-chat/agent-session-journal/journal-row-base.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' - -export function journalRowBase( - epoch: string, - seq: number, - fence: number, - ts: number, - bodies: readonly { kind: string }[] = [] -): { v: number; epoch: string; seq: number; fence: number; ts: number } { - return { v: journalRowSchemaVersion(bodies), epoch, seq, fence, ts } -} diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index a8187e95d54d..27951b308e92 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -10,6 +10,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { AgentSessionMessageSource } from '../../../shared/agent-session-message-source' import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import { agentJournalLinkageFields, namesAgentJournalProducer @@ -32,10 +33,6 @@ import { boundInlineText, DEFAULT_JOURNAL_PAYLOAD_LIMITS } from './journal-paylo import { assertSubmissionIdUnused } from './journal-write-guards' import { turnEndAfterStop } from './journal-stop-turn-end' import type { ResolveDispatchInput } from './journal-store-contracts' -import { journalRowBase } from './journal-row-base' -import { buildJournalItemRow } from './journal-item-row' -export { journalRowBase } from './journal-row-base' -export { buildJournalItemRow } from './journal-item-row' type RowBuilder = (seq: number, ts: number) => T @@ -43,12 +40,7 @@ export function journalItemRowBuilder( state: () => JournalReducerState, identity: AgentJournalItemIdentity, body: AgentJournalItemBody, - options: AgentJournalRowAttribution & { - fence: number - observedAt?: number - providerObservedAt?: number - recovered?: true - } + options: AgentJournalRowAttribution & { fence: number; observedAt?: number; recovered?: true } ): RowBuilder { return (seq, ts) => buildJournalItemRow({ @@ -58,7 +50,6 @@ export function journalItemRowBuilder( seq, fence: options.fence, ts: options.observedAt ?? ts, - providerObservedAt: options.providerObservedAt, recovered: options.recovered, linkage: options, turnScope: options.turnScope @@ -209,7 +200,7 @@ export function journalLifecycleBatchRowBuilder( * An item mutation names its own, or none to keep the row's existing one. * The reducer still reads row-level linkage as the fallback for a mutation * that names none, because a row may come from a host that wrote one. */ - options: { fence: number; recovered?: true; providerObservedAt?: number } + options: { fence: number; recovered?: true } ): RowBuilder { return (seq, ts) => { if (mutations.length === 0 || mutations.length > MAX_JOURNAL_LIFECYCLE_BATCH_MUTATIONS) { @@ -246,10 +237,7 @@ export function journalLifecycleBatchRowBuilder( ts, built.flatMap((mutation) => (mutation.kind === 'item' ? [mutation.body] : [])) ), - ...(options.recovered ? { recovered: options.recovered } : {}), - ...(options.providerObservedAt === undefined - ? {} - : { providerObservedAt: options.providerObservedAt }) + ...(options.recovered ? { recovered: options.recovered } : {}) } if (Buffer.byteLength(JSON.stringify(row), 'utf8') + 1 > MAX_JOURNAL_LIFECYCLE_BATCH_BYTES) { throw new Error('journal_lifecycle_batch_byte_bound_exceeded') @@ -258,6 +246,49 @@ export function journalLifecycleBatchRowBuilder( } } +export function journalRowBase( + epoch: string, + seq: number, + fence: number, + ts: number, + bodies: readonly { kind: string }[] = [] +): { v: number; epoch: string; seq: number; fence: number; ts: number } { + return { v: journalRowSchemaVersion(bodies), epoch, seq, fence, ts } +} + +export function buildJournalItemRow(input: { + state: JournalReducerState + identity: AgentJournalItemIdentity + body: AgentJournalItemBody + seq: number + fence: number + ts: number + recovered?: true + linkage?: AgentJournalProducerLinkage + turnScope: AgentJournalTurnScope +}): JournalItemRow { + const itemId = agentJournalItemKey(input.identity) + const resolved = input.state.aliases.get(itemId) ?? itemId + // A tombstoned row keeps its revision in `tombstones`, and the reducer drops + // any item at or below it — so a re-add has to outrank the tombstone too. + const revision = + Math.max( + input.state.items.get(resolved)?.revision ?? 0, + input.state.tombstones.get(resolved) ?? 0 + ) + 1 + const body = turnEndAfterStop(input.state, resolved, input.body) + return { + kind: 'item', + itemId, + revision, + body, + ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), + ...(input.recovered ? { recovered: input.recovered } : {}), + turnScope: input.turnScope, + ...agentJournalLinkageFields(input.linkage) + } +} + export function buildJournalTombstoneRow(input: { state: JournalReducerState itemId: string diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts index 93a3bc7b4ec6..b72a24d70bb3 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.test.ts @@ -22,48 +22,6 @@ function parse(row: Record): boolean { } describe('journal row validation', () => { - it.each([null, 'later', -1, {}])( - 'drops unusable provider receipt %j without dropping output', - (providerObservedAt) => { - const parsed = parseJournalRow( - JSON.stringify({ - ...BASE, - kind: 'item', - itemId: 'i-1', - revision: 1, - providerObservedAt, - body: { kind: 'status', text: 'provider output' } - }) - ) - expect(parsed).toEqual({ - ok: true, - row: { - ...BASE, - v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, - kind: 'item', - itemId: 'i-1', - revision: 1, - body: { kind: 'status', text: 'provider output' } - } - }) - } - ) - - it('keeps a valid optional receipt on a same-version row', () => { - const row = { - ...BASE, - kind: 'item', - itemId: 'i-1', - revision: 1, - providerObservedAt: 5_500, - body: { kind: 'status', text: 'provider output' } - } - expect(parseJournalRow(JSON.stringify(row))).toEqual({ - ok: true, - row: { ...row, v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION } - }) - }) - it('upcasts v1 rows to the current schema without changing their body', () => { const parsed = parseJournalRow( JSON.stringify({ diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index f647c073b3e7..1d226761ae25 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -55,8 +55,6 @@ type JournalRowBase = AgentJournalProducerLinkage & { fence: number /** Observed (provider or host) timestamp. Ordering is by `seq`, not by this. */ ts: number - /** Receipt of live provider output, distinct from an item's pinned start timestamp. */ - providerObservedAt?: number /** Set when crash reconciliation appended the row after the fact. */ recovered?: true /** Which turn the item this row creates belongs to. Rides the base, and is not a `v` bump, diff --git a/src/main/native-chat/agent-session-journal/journal-row-table.ts b/src/main/native-chat/agent-session-journal/journal-row-table.ts index 9074ae0019e6..7ff185857cf6 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-table.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-table.ts @@ -39,7 +39,8 @@ export function publishJournalSessionEpoch( export function insertJournalRow( db: Database.Database, sessionId: string, - row: JournalRow + row: JournalRow, + savedAt = row.ts ): number { const rowJson = serializeJournalRow(row) // A row the reader rejects would fail the chat's next load, so it is never written: the throw @@ -51,7 +52,8 @@ export function insertJournalRow( `the ${row.kind} row for ${sessionId} would not read back, so it was not written` ) } - db.prepare(INSERT_ROW).run(sessionId, row.epoch, row.seq, row.ts, rowJson) + // The table timestamp records the save; the JSON timestamp can remain pinned to item start. + db.prepare(INSERT_ROW).run(sessionId, row.epoch, row.seq, savedAt, rowJson) return Buffer.byteLength(rowJson, 'utf8') } diff --git a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts index 130226cf14e1..6c8811eb4a5b 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts @@ -6,14 +6,6 @@ import { isAdmissibleAgentSessionContextUsage } from '../../../shared/agent-sess export function dropUnusableRowAnnotations(record: Record): void { dropUnusableProducerLinkage(record) dropUnusableTurnScope(record) - if ( - record.providerObservedAt !== undefined && - (typeof record.providerObservedAt !== 'number' || - !Number.isFinite(record.providerObservedAt) || - record.providerObservedAt < 0) - ) { - delete record.providerObservedAt - } if (record.kind === 'lifecycle-batch' && Array.isArray(record.mutations)) { for (const mutation of record.mutations) { if (isPlainObject(mutation)) { diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts index 3671ce46cfcf..169ea764a9fa 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts @@ -13,6 +13,7 @@ import type { JournalHostDatabase } from './journal-host-database' import type { JournalRow } from './journal-row-schema' import { JournalRowWriter } from './journal-row-writer' import { JournalWriteQueue } from './journal-write-queue' +import { readJournalRowsAfter } from './journal-row-table' import { openTestJournalHostDatabase, readTestJournalRows, @@ -106,4 +107,21 @@ describe('journal row writer', () => { [2] ) }) + + it.each(['single', 'batch'] as const)( + 'keeps the save time separate from a pinned JSON timestamp in a %s write', + async (kind) => { + const { writer, committedRows } = writerHarness() + const pinned = (seq: number) => row(seq, 0) + await (kind === 'single' + ? writer.enqueue(pinned) + : writer.enqueueRows(() => [pinned, pinned])) + + const saved = readJournalRowsAfter(database.db, SESSION_ID, EPOCH, 0) + expect(saved).toHaveLength(kind === 'single' ? 1 : 2) + expect(saved.map((entry) => entry.ts)).toEqual(saved.map(() => 1)) + expect(saved.map((entry) => JSON.parse(entry.rowJson))).toEqual(committedRows) + expect(committedRows.map((entry) => entry.ts)).toEqual(saved.map(() => 0)) + } + ) }) diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index 60db6e0cbd1e..759acab177dc 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -27,7 +27,7 @@ export type JournalRowWriterDeps = { readOnly: () => boolean highestFence: () => number nextSequence: () => number - commit: (row: JournalRow) => void + commit: (row: JournalRow, savedAt: number) => void /** Standing hook run for EVERY appended row — the queued-draft returned * transition rides here so no rejection path can bypass it. Bookkeeping: it * runs in its own savepoint, so its failure is reported and never vetoes the row. */ @@ -48,12 +48,13 @@ export class JournalRowWriter { ): Promise { return this.deps.serialize(() => { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) - const row = build(this.deps.nextSequence(), this.deps.now()) + const savedAt = this.deps.now() + const row = build(this.deps.nextSequence(), savedAt) assertJournalFence(row.fence, this.deps.highestFence()) try { // One INSERT: the chat's epoch pointer moves only when the epoch does. this.deps.database().transaction((db) => { - insertJournalRow(db, this.deps.sessionId, row) + insertJournalRow(db, this.deps.sessionId, row, savedAt) hook?.(db, row) receipt?.write(db) this.runBookkeeping(db, row) @@ -66,7 +67,7 @@ export class JournalRowWriter { // fail. Rejecting here instead would leave the next append reusing a // sequence the table already holds. The ledger first: it cannot throw, the fold can. receipt?.committed() - this.deps.commit(row) + this.deps.commit(row, savedAt) return row }) } @@ -94,7 +95,7 @@ export class JournalRowWriter { try { this.deps.database().transaction((db) => { for (const row of rows) { - insertJournalRow(db, this.deps.sessionId, row) + insertJournalRow(db, this.deps.sessionId, row, ts) this.runBookkeeping(db, row) } }) @@ -103,7 +104,7 @@ export class JournalRowWriter { throw error } for (const row of rows) { - this.deps.commit(row) + this.deps.commit(row, ts) } return rows } diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 5acba78506a4..5425158d866f 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -40,7 +40,7 @@ export type JournalStoreHost = { readOnly: () => boolean cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void - commit: (row: JournalRow) => void + commit: (row: JournalRow, savedAt: number) => void journal: () => AgentSessionJournal enqueue: (build: (seq: number, ts: number) => JournalRow) => Promise } diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index c41fcec55322..41b892adf348 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -57,7 +57,6 @@ export type JournalAppendResult = { export type JournalItemAppendOptions = AgentJournalRowAttribution & { fence: number observedAt?: number - providerObservedAt?: number recovered?: true } export type JournalTombstoneInput = { fence: number } @@ -75,7 +74,6 @@ export type JournalLifecycleBatchInput = { mutations: readonly JournalLifecycleMutationInput[] fence: number recovered?: true - providerObservedAt?: number /** Rejects the sends still queued with this first, in the same append: a failed start's row * follows the messages it failed, and no reader meets one without the other. With none still * queued, the batch is not written either. */ diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 13da0881670e..79b9bb01e3cf 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -121,8 +121,8 @@ export class AgentSessionJournal { this.adoptLoadedJournal(loaded) this.onCommitted?.() }, - commit: (row) => { - applyJournalRow(this.state, row) + commit: (row, savedAt) => { + applyJournalRow(this.state, row, savedAt) this.onCommitted?.() }, notifyCommitted: () => this.onCommitted?.(), @@ -240,7 +240,7 @@ export class AgentSessionJournal { /** Fence of the writer that created the item, while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) - /** Provider output at this fence, excluding client actions and recovery bookkeeping. */ + /** Latest saved output at this fence, excluding client actions and recovery bookkeeping. */ lastProviderActivityAt = (fence: number): number | undefined => this.state.providerActivityAt.get(fence) diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts index f61767a0e0a1..cee9087f240a 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler.ts @@ -53,7 +53,7 @@ export type ProviderTimelineAssembler = { /** The host already wrote this command's running turn; the provider ends that same row. */ beginCommand(command: StructuredAgentSessionCommandRun): void forgetCommand(turnId: string): void - apply(event: ProviderTimelineEvent, providerObservedAt?: number): ProviderTimelineApplyResult + apply(event: ProviderTimelineEvent): ProviderTimelineApplyResult /** The turn id of the open turn, as its row and a client's Stop name it. */ readonly openTurnId: string | null /** Writes the text the coalescing window holds. */ @@ -103,15 +103,11 @@ export function createProviderTimelineAssembler( ...(deps.schedule ? { schedule: deps.schedule } : {}) }) - const textHost = ( - journal: ProviderTimelineTextHost['journal'], - providerObservedAt: number - ): ProviderTimelineTextHost => ({ + const textHost = (journal: ProviderTimelineTextHost['journal']): ProviderTimelineTextHost => ({ sink: deps.sink, state, streams, journal, - providerObservedAt, admits: (hold) => admits(hold, journal) }) @@ -120,8 +116,7 @@ export function createProviderTimelineAssembler( const applyDecided = ( event: ProviderTimelineDecidedEvent, - journal: ProviderTimelineTextHost['journal'], - providerObservedAt: number + journal: ProviderTimelineTextHost['journal'] ): ProviderTimelineApplyResult => { const serials = state.serials() const decision = decideProviderTimelineEvent( @@ -134,7 +129,7 @@ export function createProviderTimelineAssembler( if (decision.hold && !admits(decision.hold, journal)) { return { admission: PROVIDER_TIMELINE_OVER_BUDGET } } - const plan = new ProviderTimelinePlan(providerObservedAt) + const plan = new ProviderTimelinePlan() planProviderTimelineBarrier({ streams, state }, plan, event, decision) planProviderTimelineWrites(context, plan, decision, serials.next) plan.onAdmitted(() => { @@ -156,30 +151,26 @@ export function createProviderTimelineAssembler( * that stops its text and cancels its prompts; refused, the event that found it is refused with * it and its retry finds it again. */ const endSettledTurn = ( - journal: NonNullable, - providerObservedAt: number + journal: NonNullable ): ProviderTimelineApplyResult | null => { const open = state.open if (state.ended || !open || providerTimelineTurnRowState(journal, open.itemId) !== 'settled') { return null } - return applyDecided({ type: 'turn.settled', turn: open }, journal, providerObservedAt) + return applyDecided({ type: 'turn.settled', turn: open }, journal) } - const apply = ( - event: ProviderTimelineEvent, - providerObservedAt = Date.now() - ): ProviderTimelineApplyResult => { + const apply = (event: ProviderTimelineEvent): ProviderTimelineApplyResult => { const journal = deps.sink.journalItems() - const settled = journal ? endSettledTurn(journal, providerObservedAt) : null + const settled = journal ? endSettledTurn(journal) : null if (settled && !settled.admission.accepted) { return settled } switch (event.type) { case 'text.delta': - return applyProviderTimelineTextDelta(textHost(journal, providerObservedAt), event) + return applyProviderTimelineTextDelta(textHost(journal), event) case 'text.close': - return applyProviderTimelineTextClose(textHost(journal, providerObservedAt), event) + return applyProviderTimelineTextClose(textHost(journal), event) case 'activity': { const open = state.open if (state.ended || !open) { @@ -201,7 +192,7 @@ export function createProviderTimelineAssembler( case 'context.usage': case 'provider.frame': case 'session.ended': - return applyDecided(event, journal, providerObservedAt) + return applyDecided(event, journal) } } diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts index 4f70041ac2ef..f9e9e0e33e38 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-plan.ts @@ -28,19 +28,13 @@ export class ProviderTimelinePlan { private readonly admitted: (() => void)[] = [] private lifecycle = false - constructor(readonly providerObservedAt: number) {} - item(step: Omit, lifecycle = false): void { - this.steps.push({ - kind: 'item', - ...step, - options: { providerObservedAt: this.providerObservedAt, ...step.options } - }) + this.steps.push({ kind: 'item', ...step }) this.lifecycle ||= lifecycle } settlement(step: Omit): void { - this.steps.push({ kind: 'settlement', providerObservedAt: this.providerObservedAt, ...step }) + this.steps.push({ kind: 'settlement', ...step }) this.lifecycle = true } diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts index 457a4ca56726..ec93367efd48 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-text-events.ts @@ -24,7 +24,6 @@ export type ProviderTimelineTextHost = { state: ProviderTimelineState streams: ProviderTimelineTextStreams journal: StructuredAgentSessionTransitionJournal | null - providerObservedAt: number admits(hold: ProviderTimelineHold): boolean } @@ -38,7 +37,7 @@ export function applyProviderTimelineTextDelta( if (state.ended) { return { admission: ADMITTED, dropped: 'session-ended' } } - const plan = new ProviderTimelinePlan(host.providerObservedAt) + const plan = new ProviderTimelinePlan() const key = streams.key(event.item, event.join) let stream = streams.get(key) if (stream && !streams.continues(stream, event.channel, event.producer)) { @@ -65,7 +64,6 @@ export function applyProviderTimelineTextDelta( channel: event.channel, producer: event.producer, state, - providerObservedAt: host.providerObservedAt, serial: serials.next() }) if (!host.admits({ key: stream.key, bytes: stream.bytes })) { @@ -107,7 +105,7 @@ export function applyProviderTimelineTextClose( dropped: streams.stoppedFor(key, event.join) ? 'turn-settled' : 'stream-unknown' } } - const plan = new ProviderTimelinePlan(host.providerObservedAt) + const plan = new ProviderTimelinePlan() streams.planFlush(plan, stream) streams.planClose(plan, stream, event.text) if (stream.named) { diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts index 366b5e76c5c2..eda78997e6d2 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-text-streams.ts @@ -60,8 +60,6 @@ export type ProviderTimelineStream = { named: boolean channel: ProviderTimelineTextChannel producer: AgentJournalProducerLinkage | undefined - startedAt: number - providerObservedAt: number /** Whether any text was written; a whitespace-only stream completes nothing. */ written: boolean bytes: number @@ -137,7 +135,6 @@ export class ProviderTimelineTextStreams { channel: ProviderTimelineTextChannel producer: AgentJournalProducerLinkage | undefined state: ProviderTimelineState - providerObservedAt: number serial: number }): ProviderTimelineStream { const { rows } = this.deps.context @@ -152,8 +149,6 @@ export class ProviderTimelineTextStreams { named, channel: input.channel, producer: input.producer, - startedAt: input.providerObservedAt, - providerObservedAt: input.providerObservedAt, written: false, bytes: providerTimelineEntryBytes({ key: 'id' in input.item ? input.item.id : input.item.stream, @@ -180,7 +175,6 @@ export class ProviderTimelineTextStreams { this.streams.set(stream.key, stream) this.byId.set(stream.id, stream) } - stream.providerObservedAt = Math.max(stream.providerObservedAt, plan.providerObservedAt) this.coalescer.append(stream.id, text) }) } @@ -248,7 +242,7 @@ export class ProviderTimelineTextStreams { } } else { const text = boundInlineText(finalText, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text - this.planText(plan, stream, text, true, plan.providerObservedAt) + this.planText(plan, stream, text, true) } this.planRelease(plan, (each) => each === stream) } @@ -270,7 +264,7 @@ export class ProviderTimelineTextStreams { if (!stream) { return true } - const plan = new ProviderTimelinePlan(stream.providerObservedAt) + const plan = new ProviderTimelinePlan() this.planText(plan, stream, text) const admission = plan.submit(this.deps.sink) // A failed or closed sink can never take it; the session's end owns what it leaves. @@ -281,8 +275,7 @@ export class ProviderTimelineTextStreams { plan: ProviderTimelinePlan, stream: ProviderTimelineStream, text: string, - providerFinal = false, - providerObservedAt = stream.providerObservedAt + providerFinal = false ): void { const empty = providerFinal ? text.length === 0 : text.trim().length === 0 if (!stream.written && empty) { @@ -293,12 +286,7 @@ export class ProviderTimelineTextStreams { plan.item({ reservedBytes: estimateStructuredAgentSessionItemBytes(stream.row.identity, body), resolve: (journal) => this.resolveText(stream, body, journal), - options: { - ...stream.producer, - turnScope: stream.scope, - observedAt: stream.startedAt, - providerObservedAt - } + options: { ...stream.producer, turnScope: stream.scope } }) plan.onAdmitted(() => { stream.written = true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts index a0f2d8485a0d..476089292a93 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts @@ -120,23 +120,6 @@ function target( } describe('deferred structured agent-session event sink', () => { - it('records receipt before a deferred append waits for its journal', async () => { - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) - try { - deferred.sink.appendItem(identity(1), BODY, { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) - now.mockReturnValue(9_000) - deferred.bind(target(5, [])) - await deferred.drained() - expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, providerObservedAt: 1_000 } - ]) - } finally { - deferred.close() - now.mockRestore() - } - }) - it('buffers writes made before the journal exists and drains them in arrival order', async () => { const log: Recorded[] = [] const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) @@ -594,12 +577,7 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { - fence: 5, - turnScope: AGENT_JOURNAL_THREAD_SCOPE, - ...LINKAGE, - providerObservedAt: expect.any(Number) - } + { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } ]) deferred.close() } @@ -623,12 +601,7 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { - fence: 5, - turnScope: AGENT_JOURNAL_THREAD_SCOPE, - ...LINKAGE, - providerObservedAt: expect.any(Number) - } + { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } ]) deferred.close() } @@ -664,9 +637,7 @@ describe('producer linkage reaches the journal through every append path', () => // A control, not a pin. Absence is the claim, so the keys must be missing // rather than present-and-undefined: a reader holding this options object // would read `agentId: undefined` as a key that exists. - expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, providerObservedAt: expect.any(Number) } - ]) + expect(journalAppendOptions).toEqual([{ fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE }]) deferred.close() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index d3b662d72710..4a09b9fcacc9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -8,9 +8,9 @@ import type { import type { AgentSessionTurnActivity } from '../../../shared/agent-session-wire' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import { estimateStructuredAgentSessionItemBytes } from './structured-agent-session-event-sink-estimate' import { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' -import { providerObservedAppendOptions } from './structured-agent-session-journal-append-options' -import { createStructuredAgentSessionItemAppend } from './structured-agent-session-item-append' +import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' import { createStructuredAgentSessionResolvedAppend } from './structured-agent-session-resolved-append' import { createStructuredAgentSessionTransitionMembers, @@ -38,8 +38,6 @@ export type StructuredAgentSessionAppendOptions = AgentJournalProducerLinkage & lifecycle?: boolean /** Host clock to stamp on the row instead of its append time. */ observedAt?: number - /** Latest frame receipt; a coalesced checkpoint keeps this rather than its write time. */ - providerObservedAt?: number } export type StructuredAgentSessionPublishOptions = Pick< @@ -239,9 +237,8 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { settlementId: string, mutations: readonly JournalLifecycleMutationInput[], options: StructuredAgentSessionAppendOptions = {} - ): StructuredAgentSessionSinkAdmission => { - const observation = providerObservedAppendOptions(options) - return queue.submit( + ): StructuredAgentSessionSinkAdmission => + queue.submit( { bytes: Buffer.byteLength(JSON.stringify({ settlementId, mutations }), 'utf8') + 512, run: (bound) => @@ -249,13 +246,11 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { settlementId, mutations, // No row-level linkage: each mutation names its own (see the batch row builder). - fence: bound.fence, - providerObservedAt: observation.providerObservedAt + fence: bound.fence }) }, { ...options, lifecycle: true } ) - } const publish = ( options: StructuredAgentSessionPublishOptions = {} @@ -269,7 +264,23 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { options ) - const appendItem = createStructuredAgentSessionItemAppend(queue) + const appendItem: NonNullable = ( + identity, + body, + options + ) => + queue.submit( + { + bytes: estimateStructuredAgentSessionItemBytes(identity, body), + run: (bound) => + bound.journal.appendItem( + identity, + body, + structuredAgentSessionJournalAppendOptions(bound.fence, options) + ) + }, + options + ) return { sink: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts deleted file mode 100644 index 906eb8d1c302..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-item-append.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' -import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' -import { estimateStructuredAgentSessionItemBytes } from './structured-agent-session-event-sink-estimate' -import { - providerObservedAppendOptions, - structuredAgentSessionJournalAppendOptions -} from './structured-agent-session-journal-append-options' - -export function createStructuredAgentSessionItemAppend( - queue: StructuredAgentSessionSinkQueue -): NonNullable { - return (identity, body, options) => { - const observation = providerObservedAppendOptions(options) - return queue.submit( - { - bytes: estimateStructuredAgentSessionItemBytes(identity, body), - run: (bound) => - bound.journal.appendItem( - identity, - body, - structuredAgentSessionJournalAppendOptions(bound.fence, observation) - ) - }, - options - ) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts index 3faf11db4b43..e7ce667ed2b6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts @@ -6,20 +6,7 @@ import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import type { JournalItemAppendOptions } from '../agent-session-journal/journal-store-contracts' -import type { - StructuredAgentSessionAppendOptions, - StructuredAgentSessionItemAppendOptions -} from './structured-agent-session-event-sink' - -/** Capture before admission, so a delayed journal write cannot extend provider life. */ -export function providerObservedAppendOptions( - options: T -): T & { providerObservedAt: number } { - return { - ...options, - providerObservedAt: options.providerObservedAt ?? options.observedAt ?? Date.now() - } -} +import type { StructuredAgentSessionItemAppendOptions } from './structured-agent-session-event-sink' export function structuredAgentSessionJournalAppendOptions( fence: number, @@ -28,9 +15,6 @@ export function structuredAgentSessionJournalAppendOptions( return { fence, ...(options.observedAt === undefined ? {} : { observedAt: options.observedAt }), - ...(options.providerObservedAt === undefined - ? {} - : { providerObservedAt: options.providerObservedAt }), turnScope: options.turnScope, ...agentJournalLinkageFields(options) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts index 569fbdf73b54..d3640db2a0cf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.ts @@ -8,10 +8,7 @@ import type { StructuredAgentSessionEventSink, StructuredAgentSessionRevisionJournal } from './structured-agent-session-event-sink' -import { - providerObservedAppendOptions, - structuredAgentSessionJournalAppendOptions -} from './structured-agent-session-journal-append-options' +import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' type ResolvedItem = { identity: AgentJournalItemIdentity; body: AgentJournalItemBody } @@ -35,9 +32,8 @@ export function createStructuredAgentSessionResolvedAppend( resolve: (journal: StructuredAgentSessionRevisionJournal) => ResolvedItem | null, options: StructuredAgentSessionItemAppendOptions, input: { publish: boolean; lifecycle?: true; overflow: string } - ) => { - const observation = providerObservedAppendOptions(options) - return queue.submit( + ) => + queue.submit( { bytes: reservedBytes, ...(input.lifecycle ? { lifecycle: true } : {}), @@ -55,7 +51,7 @@ export function createStructuredAgentSessionResolvedAppend( } return resolved }, - structuredAgentSessionJournalAppendOptions(bound.fence, observation) + structuredAgentSessionJournalAppendOptions(bound.fence, options) ) if (landed !== null && input.publish) { bound.publish() @@ -64,7 +60,6 @@ export function createStructuredAgentSessionResolvedAppend( }, input.lifecycle ? { lifecycle: true } : options ) - } const ITEM_OVERFLOW = 'structured agent-session resolved item exceeded its reserved size' const identityOnly = (publish: boolean) => ((identitySizeBound, body, resolveIdentity, options) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts index bd132c2759fb..3161c73a96fc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts @@ -16,6 +16,7 @@ import { } from '../agent-session-journal/journal-host-database-test-support' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { HOST_TEST_NOW, hostTestAttachParams } from './structured-agent-session-host-test-data' +import { unhandledProviderFrameJournalItem } from './unhandled-provider-frame' import { createRestTestRig, REST_TEST_CALLER, @@ -26,6 +27,7 @@ import { } from './structured-agent-session-rest-test-rig' const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 +const SAVED_AT = HOST_TEST_NOW + 60_000 const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 const TURN = { provider: 'codex' as const, threadId: THREAD, turnId: 'stream-turn', ordinal: 0 } const REPLY = { ...TURN, ordinal: 1 } @@ -35,6 +37,7 @@ let rig: RestTestRig beforeEach(async () => { rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) + vi.spyOn(Date, 'now').mockImplementation(() => rig.clock.now) }) afterEach(async () => { @@ -84,12 +87,12 @@ async function beginTurn() { return events } -async function expectInterruptedOutput(): Promise { +async function expectInterruptedOutput(completedAt = LAST_CHUNK_AT): Promise { await restartAfterCrash() await rig.host.restoreReadableSessions() const snapshot = await rig.host.journalSnapshot(SESSION) const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) - expect(turn).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) + expect(turn).toMatchObject({ state: 'interrupted', completedAt }) } describe('provider output survives host restart', () => { @@ -99,7 +102,7 @@ describe('provider output survives host restart', () => { ['plan', 'item/plan/delta'], ['reasoning', 'item/reasoning/textDelta'], ['commandExecution', 'item/commandExecution/outputDelta'] - ] as const)('times actual Codex %s deltas rather than item start', async (type, method) => { + ] as const)('times saved Codex %s output rather than item start', async (type, method) => { const events = await beginTurn() let receivedAt = HOST_TEST_NOW vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) @@ -137,6 +140,7 @@ describe('provider output survives host restart', () => { }) for (let chunk = 1; chunk <= 11; chunk += 1) { receivedAt = HOST_TEST_NOW + chunk * 500 + rig.clock.now = receivedAt expect( translator.handle({ type: 'notification', @@ -160,6 +164,9 @@ describe('provider output survives host restart', () => { JSON.stringify(item.body).includes('actual chunk 11') ) expect(output?.observedAt).toBe(HOST_TEST_NOW) + expect( + rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal.lastProviderActivityAt(1) + ).toBe(LAST_CHUNK_AT) await restartAfterCrash() await rig.host.restoreReadableSessions() const snapshot = await rig.host.journalSnapshot(SESSION) @@ -170,6 +177,7 @@ describe('provider output survives host restart', () => { it('counts streamed file-change output as later proof of provider life', async () => { const events = await beginTurn() for (let chunk = 1; chunk <= 11; chunk += 1) { + rig.clock.now = HOST_TEST_NOW + chunk * 500 const translated = codexStreamingJournalItem( { id: 'file-change-1', type: 'fileChange', changes: [{ path: 'file.ts' }] }, `patch chunk ${chunk}` @@ -191,7 +199,7 @@ describe('provider output survives host restart', () => { expect(turn).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) }) - it('keeps a delayed first checkpoint at item start without counting a later flush', async () => { + it('uses a delayed first save while keeping the bubble at item start', async () => { const events = await beginTurn() const translator = createCodexJournalTranslator({ sink: events, @@ -222,7 +230,7 @@ describe('provider output survives host restart', () => { params: { turnId: TURN.turnId, itemId: 'late-first', delta: 'first durable output' }, observedAt: LAST_CHUNK_AT }) - vi.spyOn(Date, 'now').mockReturnValue(RESTARTED_AT - 1_000) + rig.clock.now = SAVED_AT translator.flush() await rig.host.flushStreamedEvents(SESSION) translator.dispose() @@ -231,7 +239,7 @@ describe('provider output survives host restart', () => { before.items.find((item) => JSON.stringify(item.body).includes('first durable output')) ?.observedAt ).toBe(HOST_TEST_NOW) - await expectInterruptedOutput() + await expectInterruptedOutput(SAVED_AT) }) it.each(['text_delta', 'thinking_delta'] as const)( @@ -257,7 +265,7 @@ describe('provider output survives host restart', () => { } }) } - vi.spyOn(Date, 'now').mockReturnValue(RESTARTED_AT - 1_000) + rig.clock.now = SAVED_AT translator.flush() await rig.host.flushStreamedEvents(SESSION) translator.dispose() @@ -267,32 +275,41 @@ describe('provider output survives host restart', () => { if (type === 'thinking_delta') { expect(output?.observedAt).toBe(HOST_TEST_NOW) } - await expectInterruptedOutput() + await expectInterruptedOutput(SAVED_AT) } ) - it('counts a synthetic provider fallback while excluding a later host notice', async () => { + it('counts a saved provider fallback but excludes a later host notice', async () => { const events = await beginTurn() + rig.clock.now = LAST_CHUNK_AT + const fallback = unhandledProviderFrameJournalItem('codex', 'test-output', { + message: 'provider output' + }) + if (!fallback) { + throw new Error('provider output was not translated') + } events.appendItem( - { provider: 'orca', clientMessageId: 'provider-fallback' }, - { kind: 'status', text: 'provider output' }, - { turnScope: TURN_SCOPE, providerObservedAt: LAST_CHUNK_AT } + { provider: 'orca', clientMessageId: 'provider-frame:codex:acquisition:1' }, + fallback.body, + { turnScope: TURN_SCOPE } ) await rig.host.flushStreamedEvents(SESSION) const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal if (!journal) { throw new Error('journal absent') } + rig.clock.now = SAVED_AT await journal.appendItem( { provider: 'orca', clientMessageId: 'host-notice' }, { kind: 'status', text: 'host notice' }, - { fence: 1, turnScope: TURN_SCOPE, observedAt: RESTARTED_AT - 1_000 } + { fence: 1, turnScope: TURN_SCOPE } ) await expectInterruptedOutput() }) it('counts a pending approval with no assistant output before the crash', async () => { const events = await beginTurn() + rig.clock.now = LAST_CHUNK_AT events.appendItem( REPLY, { @@ -319,6 +336,7 @@ describe('provider output survives host restart', () => { it('cannot record interruption from a surviving owner before exit proof', async () => { const events = await beginTurn() + rig.clock.now = LAST_CHUNK_AT events.appendItem( REPLY, { kind: 'status', text: 'working' }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts index 8b79e0e155ff..7ba2489430d2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts @@ -38,6 +38,7 @@ let rig: RestTestRig beforeEach(async () => { rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) + vi.spyOn(Date, 'now').mockImplementation(() => rig.clock.now) }) afterEach(async () => { @@ -71,6 +72,7 @@ async function streamBeforeCrash( { turnScope: AGENT_JOURNAL_THREAD_SCOPE, observedAt: HOST_TEST_NOW } ) for (let chunk = 1; chunk <= 11; chunk += 1) { + rig.clock.now = HOST_TEST_NOW + chunk * 500 events.appendItem( REPLY, { @@ -119,7 +121,7 @@ async function expectSettledStream(): Promise { expect(completedStructuredAgentTurnSeconds(timing)).toBe(5) expect( withNativeChatCutTurnNotices(snapshot.items, { agentName: 'Codex' }).filter( - (item) => item.body.kind === 'status' + (item) => item.body.kind === 'status' && item.body.text !== 'host notice' ) ).toHaveLength(1) expect(rig.adapter.acquire).toHaveBeenCalledOnce() @@ -215,6 +217,7 @@ describe('a provider stream cut short before its next lease renewal', () => { throw new Error('the streaming journal is not open') } const later = RESTARTED_AT - 1_000 + rig.clock.now = later await journal.appendItem( { provider: 'orca', clientMessageId: 'late-client-send' }, { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'next task' }] }, @@ -240,13 +243,17 @@ describe('a provider stream cut short before its next lease renewal', () => { if (!originalReply) { throw new Error('the streamed reply is missing') } - vi.spyOn(Date, 'now').mockReturnValue(later) await journal.appendLifecycleBatch({ settlementId: 'earlier-read-recovery', fence: 1, recovered: true, mutations: [{ kind: 'item', identity: REPLY, body: originalReply, turnScope: TURN_SCOPE }] }) + await journal.appendItem( + { provider: 'orca', clientMessageId: 'late-host-notice' }, + { kind: 'status', text: 'host notice' }, + { fence: 1, turnScope: TURN_SCOPE } + ) await journal.appendItem( { ...REPLY, ordinal: 3 }, { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'another owner' }] }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts deleted file mode 100644 index 93ced01c8dd9..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-timeline-receipt.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' -import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' -import { JsonlRpcTimelineLane } from '../../jsonl-rpc/timeline-lane' -import { - closeTestJournalHostDatabases, - openTestJournalHostDatabase -} from '../agent-session-journal/journal-host-database-test-support' -import { createProviderTimelineAssembler } from '../agent-session-timeline/provider-timeline-assembler' -import { refusingSink } from '../agent-session-timeline/provider-timeline-assembler-test-support' -import { providerTimelineSink } from '../agent-session-timeline/provider-timeline-plan' -import { StructuredAgentSessionHost } from './structured-agent-session-host' -import { HOST_TEST_NOW, hostTestAttachParams } from './structured-agent-session-host-test-data' -import { - createRestTestRig, - REST_TEST_CALLER, - restTestSend, - REST_TEST_SESSION as SESSION, - type RestTestRig -} from './structured-agent-session-rest-test-rig' - -const LAST_OUTPUT_AT = HOST_TEST_NOW + 5_500 -const FLUSHED_AT = HOST_TEST_NOW + 60_000 -const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 -let rig: RestTestRig - -beforeEach(async () => { - rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) -}) - -afterEach(async () => { - vi.restoreAllMocks() - await rig.dispose() - closeTestJournalHostDatabases() -}) - -async function restartAfterCrash(): Promise { - const old = rig.host - old.stopDelivery() - const { lifetime, runtimeState, sessions } = old.collaboratorsForTests() - lifetime.idleSweep.dispose() - await runtimeState.stopLeaseRenewal() - runtimeState.currentEventSink(SESSION)?.close() - await sessions.get(SESSION)?.journal.close() - closeTestJournalHostDatabases() - rig.store = await openTestAgentSessionRecordStore(rig.root) - rig.host = new StructuredAgentSessionHost({ - ...old.deps, - store: rig.store, - journalDatabase: openTestJournalHostDatabase(rig.root), - now: () => RESTARTED_AT - }) -} - -async function providerSink() { - const attached = await rig.host.attach(REST_TEST_CALLER, hostTestAttachParams(null)) - if (!attached.ok) { - throw new Error('attach refused') - } - const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('stream', attached.fence)) - if (!sent.ok) { - throw new Error('send refused') - } - await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) - const events = rig.adapter.acquire.mock.calls[0]?.[0].events - if (!events) { - throw new Error('provider sink missing') - } - const sink = providerTimelineSink(events) - if (!sink) { - throw new Error('timeline sink missing') - } - return sink -} - -async function expectInterruptedAt(at: number): Promise { - await restartAfterCrash() - await rig.host.restoreReadableSessions() - const snapshot = await rig.host.journalSnapshot(SESSION) - const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) - expect(turn).toMatchObject({ state: 'interrupted', completedAt: at }) -} - -describe('shared provider timeline receipt before coalescing', () => { - it.each([ - ['assistant', 'flush'], - ['reasoning', 'flush'], - ['assistant', 'window'], - ['reasoning', 'window'] - ] as const)( - 'ends a crashed %s stream at its last delta rather than a delayed %s', - async (channel, trigger) => { - const sink = await providerSink() - let receivedAt = HOST_TEST_NOW - vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) - let window = () => {} - const timeline = createProviderTimelineAssembler({ - sink, - sessionId: SESSION, - agent: 'opencode', - generation: 'review-generation', - namespace: 'review-acp', - schedule: (run) => { - window = run - return () => {} - } - }) - expect( - timeline.apply({ type: 'turn.open', turn: 'timeline-turn', at: HOST_TEST_NOW }) - ).toMatchObject({ admission: { accepted: true } }) - await rig.host.flushStreamedEvents(SESSION) - for (let chunk = 1; chunk <= 11; chunk += 1) { - receivedAt = HOST_TEST_NOW + chunk * 500 - expect( - timeline.apply({ - type: 'text.delta', - item: { stream: 'reply' }, - channel, - text: `chunk ${chunk} `, - join: { turn: 'timeline-turn' } - }) - ).toMatchObject({ admission: { accepted: true } }) - } - receivedAt = FLUSHED_AT - if (trigger === 'window') { - window() - } else { - timeline.flush() - } - await rig.host.flushStreamedEvents(SESSION) - timeline.dispose() - const before = await rig.host.journalSnapshot(SESSION) - const reply = before.items.find((item) => JSON.stringify(item.body).includes('chunk 11')) - expect(reply?.observedAt).toBe(HOST_TEST_NOW + 500) - expect(rig.store.getRecord(SESSION)?.lease.lastRenewedAt).toBe(HOST_TEST_NOW) - - await expectInterruptedAt(LAST_OUTPUT_AT) - } - ) - - it.each(['assistant', 'reasoning'] as const)( - 'retains held %s delta receipts across lane and coalescer retries', - async (channel) => { - const sink = await providerSink() - let receivedAt = HOST_TEST_NOW - vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) - let blocked = true - const lane = new JsonlRpcTimelineLane({ - sink: refusingSink(sink, () => blocked), - sessionId: SESSION, - agent: 'pi', - generation: 'receipt-generation', - namespace: 'receipt-pi', - pauseReading: vi.fn(), - resumeReading: vi.fn(), - onInputAccepted: vi.fn(), - onFailed: vi.fn() - }) - try { - lane.apply([{ type: 'turn.open', turn: 'held-turn', at: HOST_TEST_NOW }]) - for (let chunk = 1; chunk <= 11; chunk += 1) { - receivedAt = HOST_TEST_NOW + chunk * 500 - lane.apply([ - { - type: 'text.delta', - item: { stream: 'reply' }, - channel, - text: `held chunk ${chunk} `, - join: { turn: 'held-turn' } - } - ]) - } - receivedAt = FLUSHED_AT - blocked = false - lane.retry() - blocked = true - lane.flush() - await rig.host.flushStreamedEvents(SESSION) - const refused = await rig.host.journalSnapshot(SESSION) - expect(refused.items.some((item) => JSON.stringify(item.body).includes('held chunk'))).toBe( - false - ) - receivedAt += 60_000 - blocked = false - lane.flush() - await rig.host.flushStreamedEvents(SESSION) - const before = await rig.host.journalSnapshot(SESSION) - const reply = before.items.find((item) => - JSON.stringify(item.body).includes('held chunk 11') - ) - expect(reply?.observedAt).toBe(HOST_TEST_NOW + 500) - } finally { - lane.dispose() - } - await expectInterruptedAt(LAST_OUTPUT_AT) - } - ) - - it('keeps a held turn start separate from its receipt and later admission', async () => { - const sink = await providerSink() - vi.spyOn(Date, 'now').mockReturnValue(LAST_OUTPUT_AT) - let blocked = true - const lane = new JsonlRpcTimelineLane({ - sink: refusingSink(sink, () => blocked), - sessionId: SESSION, - agent: 'opencode', - generation: 'held-open-generation', - namespace: 'held-open-acp', - pauseReading: vi.fn(), - resumeReading: vi.fn(), - onInputAccepted: vi.fn(), - onFailed: vi.fn() - }) - try { - lane.apply([{ type: 'turn.open', turn: 'held-turn', at: HOST_TEST_NOW }]) - vi.spyOn(Date, 'now').mockReturnValue(FLUSHED_AT) - blocked = false - lane.retry() - await rig.host.flushStreamedEvents(SESSION) - } finally { - lane.dispose() - } - await expectInterruptedAt(LAST_OUTPUT_AT) - }) - - it('records a held final text snapshot at its receipt rather than the lane retry', async () => { - const sink = await providerSink() - let receivedAt = HOST_TEST_NOW - vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) - let blocked = false - const lane = new JsonlRpcTimelineLane({ - sink: refusingSink(sink, () => blocked), - sessionId: SESSION, - agent: 'pi', - generation: 'final-generation', - namespace: 'final-pi', - pauseReading: vi.fn(), - resumeReading: vi.fn(), - onInputAccepted: vi.fn(), - onFailed: vi.fn() - }) - try { - lane.apply([{ type: 'turn.open', turn: 'held-turn', at: HOST_TEST_NOW }]) - receivedAt += 500 - lane.apply([ - { - type: 'text.delta', - item: { stream: 'reply' }, - channel: 'assistant', - text: 'partial' - } - ]) - lane.flush() - await rig.host.flushStreamedEvents(SESSION) - receivedAt = LAST_OUTPUT_AT - blocked = true - lane.apply([{ type: 'text.close', item: { stream: 'reply' }, text: 'final reply' }]) - receivedAt = FLUSHED_AT - blocked = false - lane.retry() - await rig.host.flushStreamedEvents(SESSION) - const before = await rig.host.journalSnapshot(SESSION) - const reply = before.items.find((item) => JSON.stringify(item.body).includes('final reply')) - expect(reply?.observedAt).toBe(HOST_TEST_NOW + 500) - } finally { - lane.dispose() - } - await expectInterruptedAt(LAST_OUTPUT_AT) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index 7f6868cca5b9..aa82c1c77941 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -36,7 +36,7 @@ export function turnVerdictFromDeathEvidence( evidence: AgentSessionDeathEvidence | null | undefined, /** Fence of the owner that wrote the turn. */ turnFence: number | undefined, - /** A provider observation or Stop that found the turn running: a later proof of life. */ + /** Saved provider output or a Stop that found the turn running: a later proof of life. */ liveAt?: number ): StructuredAgentSessionTurnVerdict { if (!evidence) { @@ -55,13 +55,13 @@ export function turnVerdictFromDeathEvidence( return { state: 'interrupted', completedAt: evidence.observedAt } } // A probe finds a dead child long after it died; its last renewal bounds the end, so the turn never - // counts the time Orca was down. Provider output and a Stop that found the turn running can + // counts the time Orca was down. Saved provider output and a Stop that found the turn running can // prove life after that renewal; a client's later send or a recovery write cannot. const lastAlive = Math.max(evidence.lastProvenAliveAt ?? evidence.observedAt, liveAt ?? 0) return { state: 'interrupted', completedAt: Math.min(lastAlive, evidence.observedAt) } } -/** The latest observation of this turn's owner, including a Stop that found it running. */ +/** The latest saved output from this turn's owner, including a Stop that found it running. */ export function lastProvenTurnLiveAt( journal: Pick, item: AgentJournalRenderItem diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index d2c3dc40b220..4c73941ee5dc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -21,10 +21,7 @@ import type { StructuredAgentSessionSinkAdmission } from './structured-agent-session-event-sink' import type { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' -import { - providerObservedAppendOptions, - structuredAgentSessionJournalAppendOptions -} from './structured-agent-session-journal-append-options' +import { structuredAgentSessionJournalAppendOptions } from './structured-agent-session-journal-append-options' /** What a transition step reads: rows by key, every row, and the turns they joined. */ export type StructuredAgentSessionTransitionJournal = Pick< @@ -48,7 +45,6 @@ export type StructuredAgentSessionTransitionStep = kind: 'settlement' /** Unique per settlement: the journal applies one id once. */ settlementId: string - providerObservedAt?: number /** Paces the queue only; the mutations are the journal's to choose. */ reservedBytes: number /** Read at execution; none writes nothing. */ @@ -88,17 +84,8 @@ export function createStructuredAgentSessionTransitionMembers( function transitionAppend( queue: StructuredAgentSessionSinkQueue ): (transition: StructuredAgentSessionTransition) => StructuredAgentSessionSinkAdmission { - return (transition) => { - const providerObservedAt = Date.now() - const steps = transition.steps.map((step) => - step.kind === 'item' - ? { - ...step, - options: providerObservedAppendOptions(step.options) - } - : step - ) - return queue.submit({ + return (transition) => + queue.submit({ bytes: transition.steps.reduce((total, step) => total + step.reservedBytes, 0) + (transition.publish ? 1 : 0), @@ -107,7 +94,7 @@ function transitionAppend( run: async (bound) => { const { journal, fence } = bound const wrote = await journal.appendSteps( - steps.map((step): JournalStep => + transition.steps.map((step): JournalStep => step.kind === 'item' ? { kind: 'item', @@ -129,7 +116,6 @@ function transitionAppend( batch: { settlementId: step.settlementId, fence, - providerObservedAt: step.providerObservedAt ?? providerObservedAt, resolve: () => step.resolve(journal) } } @@ -140,5 +126,4 @@ function transitionAppend( } } }) - } } From 5ca43e4a31c2d5ecb28ef0734735eb87bd515c9b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:14:45 -0700 Subject: [PATCH 07/13] fix(chat): exclude host recovery revisions from provider activity Mark journal-open and connection-cleanup revisions with the existing recovery annotation so an interrupted turn does not include host downtime. Preserve buffered provider output and advance restart fixtures to the restart clock. --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + .../journal-store-restore.ts | 8 +- .../journal-subagent-liveness.test.ts | 2 + .../provider-timeline-decision.ts | 6 +- .../provider-timeline-transition-layout.ts | 1 + .../provider-timeline-turn-decisions.ts | 1 + ...nt-session-restart-provider-output.test.ts | 2 +- ...nt-session-restart-recovery-output.test.ts | 208 ++++++++++++++++++ ...nt-session-restart-stream-duration.test.ts | 1 + .../structured-agent-session-transition.ts | 2 + 10 files changed, 228 insertions(+), 4 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 8a3e6e0468eb..cadf3038a095 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -114,6 +114,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts', diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index 0f5208dea9bb..e8cb28e6d321 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -33,9 +33,13 @@ export async function restoreJournalStore( replay: () => replayJournal(database.db, host.identity.sessionId), start: () => collaborators.epochController.start('session_created', 0), adopt: host.adopt, - // Roster notices are about the conversation, not any turn in it. + // Journal-open revisions are host recovery, not new provider output. appendItem: (identity, body, fence) => - host.journal().appendItem(identity, body, { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE }), + host.journal().appendItem(identity, body, { + fence, + turnScope: AGENT_JOURNAL_THREAD_SCOPE, + recovered: true + }), highestFence: () => host.state().highestFence }) } diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts index 4da76e0ba429..8649a0ad10ba 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts @@ -236,10 +236,12 @@ describe('journal reopen after the writing host is gone', () => { // The host dies without ever settling them — no `ended`, so no session sweep. await live.close() + clock += 3_600_000 const reopened = await open() const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) + expect(afterRestart.recovered).toBe(true) expect(rosterOf(afterRestart.body)).toMatchObject([ { id: 'a', state: 'unverifiable' }, { id: 'b', state: 'unverifiable' } diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-decision.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-decision.ts index 61fdfe4cd496..57558fc73f2d 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-decision.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-decision.ts @@ -76,7 +76,11 @@ export type ProviderTimelineDecision = { /** The event's change to the state, made when the sink admits it. */ commit?: (state: ProviderTimelineState) => void /** The settlement the event owes, read from the journal when it runs. */ - settle?: { what: string; resolve: (journal: Journal) => readonly JournalLifecycleMutationInput[] } + settle?: { + what: string + recovered?: true + resolve: (journal: Journal) => readonly JournalLifecycleMutationInput[] + } writes?: readonly ProviderTimelineItemWrite[] /** The streamed item whose text this event's full snapshot replaces. */ closes?: string diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-transition-layout.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-transition-layout.ts index f2c221ed8e81..663d8aefd140 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-transition-layout.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-transition-layout.ts @@ -79,6 +79,7 @@ export function planProviderTimelineWrites( plan.settlement({ settlementId: providerTimelineSettlementId(context, serial(), settle.what), reservedBytes: SETTLEMENT_RESERVED_BYTES, + recovered: settle.recovered, resolve: settle.resolve }) } diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-turn-decisions.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-turn-decisions.ts index e754884ab3ce..973de114862f 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-turn-decisions.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-turn-decisions.ts @@ -260,6 +260,7 @@ export function decideSessionEnd( return { settle: { what: 'session-end', + recovered: true, resolve: (journal) => providerTimelineSettlement(journal, 'session', { turns: runningProviderTimelineTurns(journal), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts index 3161c73a96fc..694443515dd9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts @@ -56,6 +56,7 @@ async function restartAfterCrash(): Promise { runtimeState.currentEventSink(SESSION)?.close() await sessions.get(SESSION)?.journal.close() closeTestJournalHostDatabases() + rig.clock.now = RESTARTED_AT rig.store = await openTestAgentSessionRecordStore(rig.root) rig.host = new StructuredAgentSessionHost({ ...old.deps, @@ -105,7 +106,6 @@ describe('provider output survives host restart', () => { ] as const)('times saved Codex %s output rather than item start', async (type, method) => { const events = await beginTurn() let receivedAt = HOST_TEST_NOW - vi.spyOn(Date, 'now').mockImplementation(() => receivedAt) const translator = createCodexJournalTranslator({ sink: events, primaryThreadId: () => THREAD, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts new file mode 100644 index 000000000000..401e98d8ae63 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts @@ -0,0 +1,208 @@ +import { StructuredAgentRegistry } from './structured-agent-registry' +import { acpLaunchSpecFor } from '../../acp/acp-launch-specs' +import { acpStructuredAgentDefinition } from '../../acp/acp-structured-agent-definitions' +import { AcpBackgroundTaskTimeline } from '../../acp/acp-background-task-timeline' +import { createProviderTimelineAssembler } from '../agent-session-timeline/provider-timeline-assembler' +import { providerTimelineSink } from '../agent-session-timeline/provider-timeline-plan' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + completedStructuredAgentTurnSeconds, + selectStructuredAgentTurnTimings +} from '../../../shared/structured-agent-session-turn-timing' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_LOCATION, + HOST_TEST_NOW, + hostTestAttachParams +} from './structured-agent-session-host-test-data' +import { + createRestTestRig, + REST_TEST_CALLER, + restTestSend, + REST_TEST_SESSION as SESSION, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 +const CONNECTION_CLOSED_AT = HOST_TEST_NOW + 60_000 +const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) + vi.spyOn(Date, 'now').mockImplementation(() => rig.clock.now) + const spec = acpLaunchSpecFor('grok') + if (!spec) { + throw new Error('ACP agent definition absent') + } + rig.host.deps.agents = new StructuredAgentRegistry([ + { definition: acpStructuredAgentDefinition(spec), adapter: rig.host.deps.adapter } + ]) + rig.adapter.acquire.mockImplementation(async ({ fence, spawnToken }) => ({ + acquisitionGeneration: 'acp-generation', + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${fence}`, + handle: { transport: 'acp', agent: 'grok', nativeId: 'acp-session' }, + origin: 'created', + mintedAtFence: fence, + observedAt: rig.clock.now + } + })) + rig.adapter.dispatch.mockImplementation(async ({ clientMessageId }) => ({ + state: 'accepted', + providerIdentity: { provider: 'orca', clientMessageId } + })) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await rig.dispose() + closeTestJournalHostDatabases() +}) + +/** Drops the old host without an exit callback or graceful turn settlement. */ +async function restartAfterCrash(): Promise { + const old = rig.host + old.stopDelivery() + const { lifetime, runtimeState, sessions } = old.collaboratorsForTests() + lifetime.idleSweep.dispose() + await runtimeState.stopLeaseRenewal() + runtimeState.currentEventSink(SESSION)?.close() + await sessions.get(SESSION)?.journal.close() + closeTestJournalHostDatabases() + rig.clock.now = RESTARTED_AT + rig.store = await openTestAgentSessionRecordStore(rig.root) + rig.host = new StructuredAgentSessionHost({ + ...old.deps, + store: rig.store, + journalDatabase: openTestJournalHostDatabase(rig.root), + now: () => RESTARTED_AT + }) +} + +async function beginTurn(workspaceKind: 'git-worktree' | 'folder') { + const attached = await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(null, { + provider: 'grok', + agent: 'grok', + providerHandle: undefined, + location: { ...HOST_TEST_LOCATION, workspaceKind }, + accountHome: { variable: 'GROK_HOME', path: rig.root } + }) + ) + if (!attached.ok) { + throw new Error('attach refused') + } + const sent = await rig.host.send( + REST_TEST_CALLER, + restTestSend('background work', attached.fence) + ) + if (!sent.ok) { + throw new Error('send refused') + } + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) + const events = rig.adapter.acquire.mock.calls[0]?.[0].events + if (!events) { + throw new Error('event sink absent') + } + expect(rig.store.getRecord(SESSION)?.provider).toBe('grok') + return events +} + +describe.each([ + ['journal-open', false], + ['session-end', false], + ['session-end', true] +] as const)('%s recovery with buffered output: %s', (recovery, bufferedOutput) => { + it.each(['git-worktree', 'folder'] as const)( + 'excludes ACP background-task recovery in a %s workspace', + async (workspaceKind) => { + const events = await beginTurn(workspaceKind) + const sink = providerTimelineSink(events) + if (!sink) { + throw new Error('timeline sink absent') + } + const assembler = createProviderTimelineAssembler({ + sink, + sessionId: SESSION, + agent: 'grok', + generation: 'acp-generation', + namespace: 'acp-session', + schedule: () => () => {} + }) + assembler.apply({ type: 'turn.open', turn: 'acp-turn', at: HOST_TEST_NOW }) + rig.clock.now = LAST_CHUNK_AT + const tasks = new AcpBackgroundTaskTimeline(() => undefined) + for (const event of tasks.translate( + [ + { + taskId: 'task-1', + kind: 'command', + label: 'sleep 20', + state: 'working', + startedAt: HOST_TEST_NOW + } + ], + { turn: 'acp-turn' } + )) { + expect(assembler.apply(event).admission).toMatchObject({ accepted: true }) + } + assembler.flush() + await rig.host.flushStreamedEvents(SESSION) + if (bufferedOutput) { + expect( + assembler.apply({ + type: 'text.delta', + item: { stream: 'tail' }, + channel: 'assistant', + text: 'buffered provider output' + }).admission + ).toMatchObject({ accepted: true }) + } + if (recovery === 'session-end') { + rig.clock.now = CONNECTION_CLOSED_AT + expect( + assembler.apply({ type: 'session.ended', verdict: { state: 'unverifiable' } }).admission + ).toMatchObject({ accepted: true }) + await rig.host.flushStreamedEvents(SESSION) + } + assembler.dispose() + const journal = rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal + if (!journal) { + throw new Error('journal absent') + } + const completedAt = bufferedOutput ? CONNECTION_CLOSED_AT : LAST_CHUNK_AT + expect(journal.lastProviderActivityAt(1)).toBe(completedAt) + await restartAfterCrash() + await rig.host.restoreReadableSessions() + const snapshot = await rig.host.journalSnapshot(SESSION) + const background = snapshot.items.find( + (item) => + item.body.kind === 'message' && + item.body.blocks.some((block) => block.type === 'background-task') + ) + expect(background?.body).toMatchObject({ + blocks: expect.arrayContaining([expect.objectContaining({ state: 'unverifiable' })]) + }) + const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + expect(turn).toMatchObject({ state: 'interrupted', completedAt }) + const [timing] = selectStructuredAgentTurnTimings(snapshot.items).values() + expect(completedStructuredAgentTurnSeconds(timing)).toBe(bufferedOutput ? 60 : 5) + expect( + rig.host.collaboratorsForTests().sessions.get(SESSION)?.journal.lastProviderActivityAt(1) + ).toBe(completedAt) + await restartAfterCrash() + await rig.host.restoreReadableSessions() + expect((await rig.host.journalSnapshot(SESSION)).items).toEqual(snapshot.items) + } + ) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts index 7ba2489430d2..616fce6a3a68 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts @@ -98,6 +98,7 @@ async function restartAfterCrash(): Promise { runtimeState.currentEventSink(SESSION)?.close() await sessions.get(SESSION)?.journal.close() closeTestJournalHostDatabases() + rig.clock.now = RESTARTED_AT rig.store = await openTestAgentSessionRecordStore(rig.root) rig.host = new StructuredAgentSessionHost({ ...old.deps, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index 4c73941ee5dc..ba1c7ec2c4ce 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -45,6 +45,7 @@ export type StructuredAgentSessionTransitionStep = kind: 'settlement' /** Unique per settlement: the journal applies one id once. */ settlementId: string + recovered?: true /** Paces the queue only; the mutations are the journal's to choose. */ reservedBytes: number /** Read at execution; none writes nothing. */ @@ -116,6 +117,7 @@ function transitionAppend( batch: { settlementId: step.settlementId, fence, + recovered: step.recovered, resolve: () => step.resolve(journal) } } From 876915c711882a0250278014616efd41dd6232c1 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:53:39 -0700 Subject: [PATCH 08/13] fix(native-chat): interrupt turns a replaced server left running When the managed server died mid-reply and a new one started on the same host, the cut-off turn settled `unverifiable` with no end whenever the old provider's identity could not be proven gone, and nothing ever revised it: the chat showed no turn status and the sidebar said "unconfirmed". The new server opens the chat records holding the host's instance lock, so every runtime that wrote them before is gone, and every provider it held lost its pipes with it. Record that at load (the fences those runtimes granted, the loaded owner's last renewal, how its runtime ended) and let the stale settle end those runtimes' running or unverifiable turns as interrupted at the owner's last saved output or renewal, with the existing Orca-stop row. Lease adjudication and the live / unverifiable / exited owner verdicts are unchanged. --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + ...ured-agent-session-attach-orchestration.ts | 1 + ...uctured-agent-session-conversation-open.ts | 10 +- ...tured-agent-session-crash-turn-end.test.ts | 86 ++++--- ...gent-session-dead-generation-settlement.ts | 80 ++++--- ...ed-agent-session-readable-restorer.test.ts | 6 +- ...agent-session-replaced-server-turn.test.ts | 224 ++++++++++++++++++ ...ured-agent-session-restart-restore.test.ts | 2 +- ...nt-session-restart-stream-duration.test.ts | 3 +- ...structured-agent-session-stale-stop-row.ts | 68 ++++++ ...ctured-agent-session-stale-turn-verdict.ts | 60 +++-- ...t-session-wedged-profile-migration.test.ts | 8 +- .../runtime/agent-session-record-store.ts | 9 +- .../agent-session-replaced-runtime.test.ts | 54 +++++ .../runtime/agent-session-replaced-runtime.ts | 43 ++++ .../agent-session-store-transactions.ts | 13 +- 16 files changed, 556 insertions(+), 112 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-stale-stop-row.ts create mode 100644 src/main/runtime/agent-session-replaced-runtime.test.ts create mode 100644 src/main/runtime/agent-session-replaced-runtime.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index cadf3038a095..7e30f9eaf999 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -110,6 +110,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-repeat-press.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-repeated-stop.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-resend-answer.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-resolved-append.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index e801a99042da..e72f1a9540bd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -222,6 +222,7 @@ async function runAttachUnderAbort( fence, acquisitionGeneration, deathEvidence: priorDeathEvidence, + replaced: context.deps.store.replacedRuntime(sessionId), failureTextContext: structuredAgentSessionFailureWordsContext(priorRecord) }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index ec7f478501f7..3dbe79ed697a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -38,7 +38,7 @@ export type OpenedStructuredAgentSessionConversation = { } export type StructuredAgentSessionConversationOpenDeps = { - store: Pick + store: Pick journalDatabase: JournalHostDatabase logger: StructuredAgentSessionHostDeps['logger'] } @@ -81,7 +81,9 @@ export async function openStructuredAgentSessionConversation( /** The open itself, indexed by nobody yet: the caller adopts the result. */ export async function openStructuredAgentSessionConversationJournal( - deps: Omit, + /** An acquisition's open settles nothing itself, so it needs no store. */ + deps: Omit & + Partial>, record: AgentSessionRecord, options: StructuredAgentSessionConversationOpenOptions = {} ): Promise { @@ -148,7 +150,8 @@ export async function resettleOpenStructuredAgentSessionConversation( } async function settleGoneGeneration( - deps: Pick, + deps: Pick & + Partial>, record: AgentSessionRecord, journal: AgentSessionJournal ): Promise { @@ -159,6 +162,7 @@ async function settleGoneGeneration( fence: record.lease.runtimeFence, acquisitionGeneration: null, deathEvidence: record.lease.deathEvidence ?? null, + replaced: deps.store?.replacedRuntime(record.sessionId), failureTextContext: structuredAgentSessionFailureWordsContext(record) }) } catch (error) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index a36518c5e614..cd9ecc7e7418 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -32,6 +32,8 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { AgentSessionAcquisitionExitUnprovenError, type StructuredAgentSessionAdapter @@ -181,6 +183,9 @@ function openHost(overrides: Partial): void { }) } +/** The turn ended where its owner was last proven alive, whatever proof ended it. */ +const INTERRUPTED_TURN = { turnId: 'turn-1', state: 'interrupted', completedAt: LAST_RENEWED_AT } + /** What a client reads over the wire. */ async function settledTurn() { return (await host.journalSnapshot(SESSION)).items @@ -195,6 +200,7 @@ beforeEach(async () => { }) afterEach(async () => { + vi.restoreAllMocks() await host?.flushAllStreamedEvents() closeTestJournalHostDatabases() await rm(root, { recursive: true, force: true }) @@ -302,10 +308,11 @@ function drainSession(): Promise { return host.collaboratorsForTests().serialize(SESSION, async () => {}) } -// On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner, -// so the open can only call the turn unverifiable; the proof, whoever writes it, then revises it. +// On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner. +// The Orca that held the owner is gone (this one replaced it), so the open itself ends the turn; the +// probe's later proof has nothing left to revise. describe('a turn a read reached before the reconcile proved its owner dead', () => { - it('reads unverifiable, then interrupted at the last renewal, and a subscriber is sent both', async () => { + it('reads interrupted at the last renewal from the first open, and a subscriber is sent it once', async () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) const events: AgentSessionSubscribeEvent[] = [] const unsubscribe = await host.subscribe({ @@ -313,18 +320,12 @@ describe('a turn a read reached before the reconcile proved its owner dead', () sessionId: SESSION, emit: (event) => events.push(event) }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) await host.reconcileRestartLeases() await drainSession() - // A paired or phone client learns of the revision through the ordinary journal publish. - await vi.waitFor(() => - expect(turnStatesSent(events)).toMatchObject([ - { state: 'unverifiable' }, - { state: 'interrupted', completedAt: LAST_RENEWED_AT } - ]) - ) + expect(turnStatesSent(events)).toMatchObject([INTERRUPTED_TURN]) const { items } = await host.journalSnapshot(SESSION) const [timing] = selectStructuredAgentTurnTimings(items).values() expect(completedStructuredAgentTurnSeconds(timing)).toBe(27) @@ -332,7 +333,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () unsubscribe() }) - it('reports the revision to the status feed as an interruption, which the chat folds as worked', async () => { + it('reports the open to the status feed as an interruption, which the chat folds as worked', async () => { const published: AgentSessionStatusSummary[] = [] openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }), @@ -343,13 +344,13 @@ describe('a turn a read reached before the reconcile proved its owner dead', () published .filter((summary) => summary.sessionId === SESSION && summary.turnOutcome) .map((summary) => summary.turnOutcome) - expect(outcomes().at(-1)).toBe('unconfirmed') + // The sidebar's red Failed until seen; the turn folds as "Worked for 27s" beside its notice row. + expect(outcomes()).toEqual(['interruption']) await host.reconcileRestartLeases() await drainSession() - // The sidebar's red Failed until seen; the turn folds as "Worked for 27s" beside its notice row. - await vi.waitFor(() => expect(outcomes().at(-1)).toBe('interruption')) + expect(outcomes().at(-1)).toBe('interruption') const [timing] = selectStructuredAgentTurnTimings( (await host.journalSnapshot(SESSION)).items ).values() @@ -362,7 +363,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () ).toEqual({ key: 'workedFor', duration: '27s' }) }) - it('explains the turn its proof revised, so the cut reads once and offers Continue', async () => { + it('explains the turn it cut, so the cut reads once and offers Continue', async () => { // The Orca that crashed recorded that it started, and never that it ended. const crashed = crashedClaudeRecord() const ownerProcess = crashed.lease.ownerProcess @@ -380,7 +381,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () beginAgentSessionRuntimeRecord(root, 'runtime-crashed', TOOL_STARTED_AT - 60_000) store = await openTestAgentSessionRecordStore(root) openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) await host.reconcileRestartLeases() await drainSession() @@ -418,7 +419,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () expect(await host.journalSnapshot(SESSION)).toEqual(settled) }) - it('revises it when recovery stops a child that outlived Orca, with no send', async () => { + it('ends it at the last renewal although recovery stops a child that outlived Orca later', async () => { let alive = true const probeOwner = async (): Promise => alive @@ -441,7 +442,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () stopOwnerProcess }) await host.history({ sessionId: SESSION, direction: 'tail' }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) // The reconcile only parks the live orphan in recovery; recovery's stop is what proves it gone. await host.restoreReadableSessions() @@ -510,31 +511,32 @@ describe('a turn a read reached before the reconcile proved its owner dead', () ]) }) - it('stays unverifiable when the revision cannot be written, and a later open revises it', async () => { + it('stays running when the open cannot write its settle, and a later open settles it', async () => { let now = RELAUNCHED_AT const log = recordingStructuredAgentSessionLogger() - openHost({ - probeOwner: async () => ({ outcome: 'pid-absent' }), - now: () => now, - logger: log.logger + const append = AgentSessionJournal.prototype.appendLifecycleBatch + let refused = false + vi.spyOn(AgentSessionJournal.prototype, 'appendLifecycleBatch').mockImplementation(function ( + this: AgentSessionJournal, + input + ) { + if (!refused && input.settlementId.startsWith(STALE_SESSION_ROW_PREFIX)) { + refused = true + return Promise.reject(new Error('disk full')) + } + return append.call(this, input) }) - await host.history({ sessionId: SESSION, direction: 'tail' }) - const { journal } = host.collaboratorsForTests().sessions.get(SESSION)! - vi.spyOn(journal, 'appendLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) + openHost({ now: () => now, logger: log.logger }) - await host.reconcileRestartLeases() - await drainSession() + await host.history({ sessionId: SESSION, direction: 'tail' }) expect(log.scopes()).toEqual(['open-dead-generation']) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) - // The proof is durable on the record, so the next open converges. + expect(await settledTurn()).toMatchObject({ turnId: 'turn-1', state: 'running' }) + // Nothing is owed in between: the next open derives the same end again. now += STRUCTURED_AGENT_SESSION_IDLE_MS + 1 await host.collaboratorsForTests().lifetime.idleSweep.tick() expect(host.hasSession(SESSION)).toBe(false) - expect(await settledTurn()).toMatchObject({ - state: 'interrupted', - completedAt: LAST_RENEWED_AT - }) + expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) }) }) @@ -583,15 +585,9 @@ async function hostWithFailingFirstStart(failure: Error) { } } -/** The turn as the crashed owner left it, with no end: nothing proves when that owner stopped. */ -const UNVERIFIABLE_TURN = { - turnId: 'turn-1', - state: 'unverifiable', - startedAt: TOOL_STARTED_AT - 2_000 -} - // The relaunch proved the fence-13 owner gone, but a start reserving fence 15 clears that proof -// before the turn is settled; what the record holds afterwards is about the start's own child. +// before the turn is settled; what the record holds afterwards is about the start's own child. The +// turn is judged by the runtime this one replaced, at its owner's last renewal, never by that death. describe('a turn a newer start could not settle before it failed', () => { it('is not judged by the death of the child it left for recovery', async () => { const { attach, advance } = await hostWithFailingFirstStart( @@ -613,7 +609,7 @@ describe('a turn a newer start could not settle before it failed', () => { }) await expect(attach(16)).resolves.toMatchObject({ ok: true }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) }) it('is not judged by the watched exit of a start that failed', async () => { @@ -629,6 +625,6 @@ describe('a turn a newer start could not settle before it failed', () => { await expect(attach(16)).resolves.toMatchObject({ ok: true }) // Main ended it at the failed start, an hour after the crash, with the start's reason. - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 2c3d1849cb55..182190f336e5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -39,6 +39,7 @@ import { runningTurnLifecycleRevisions, lastProvenTurnLiveAt, turnVerdictFromDeathEvidence, + turnVerdictFromReplacedRuntime, watchedExitRevisions, type StructuredAgentSessionTurnVerdict, type StructuredAgentSessionWatchedExit @@ -47,13 +48,18 @@ import { exitedRootTurnScope, settledRootTurnScope } from './structured-agent-session-exit-turn-scope' -import { orcaStopRowBody } from './structured-agent-session-orca-stop-row' import { hasUnfinishedStructuredAgentSessionWork, isInProgressStructuredAgentSessionItem, type DeadGenerationJournal } from './structured-agent-session-unfinished-work' import { withdrawCodexSendsNoTurnOpenedFor } from './structured-agent-session-unopened-send-withdrawal' +import type { AgentSessionReplacedRuntime } from '../../runtime/agent-session-replaced-runtime' +import { + isUnverifiableTurn, + staleStopRow, + type StaleCut +} from './structured-agent-session-stale-stop-row' /** Bounds the exit reason the lease keeps as log evidence; a provider diagnostic is held to the * same cap. */ @@ -175,11 +181,11 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { /** * Settles whatever a generation with no child in this process left running: found when a new child - * is acquired, or when a chat is reopened for reading. Derived from the journal and the lease's - * death evidence each time, so nothing is owed in between. Proven death ends the turn interrupted, - * and a proof written after an earlier settle revises what that settle left `unverifiable`, the - * turn and the calls it closed alike. Must run before a new child's buffered events land, or a - * live turn would be judged. + * is acquired, or when a chat is reopened for reading. Derived from the journal, the lease's death + * evidence and the runtimes this one replaced each time, so nothing is owed in between. Either proof + * ends the turn interrupted, and a proof written after an earlier settle revises what that settle + * left `unverifiable`, the turn and the calls it closed alike. Must run before a new child's + * buffered events land, or a live turn would be judged. */ export async function settleStaleStructuredAgentSessionState(input: { journal: AgentSessionJournal @@ -187,23 +193,35 @@ export async function settleStaleStructuredAgentSessionState(input: { fence: number acquisitionGeneration: string | null deathEvidence: AgentSessionDeathEvidence | null + /** The runtimes this one replaced for this chat: every owner they held lost its pipes with them. */ + replaced?: AgentSessionReplacedRuntime /** Who the exit row names. */ failureTextContext?: AgentSessionFailureWordsContext }): Promise { - const { journal } = input + const { journal, replaced } = input const items = journal.snapshot().items // Each turn is judged by the evidence only if it names that turn's owner. - const verdictFor = (item: AgentJournalRenderItem) => + const deathVerdictFor = (item: AgentJournalRenderItem) => turnVerdictFromDeathEvidence( input.deathEvidence, journal.itemFence(item.itemId), lastProvenTurnLiveAt(journal, item) ) + const replacedVerdictFor = (item: AgentJournalRenderItem) => + turnVerdictFromReplacedRuntime( + replaced, + journal.itemFence(item.itemId), + lastProvenTurnLiveAt(journal, item) + ) + const verdictFor = (item: AgentJournalRenderItem) => { + const byDeath = deathVerdictFor(item) + return byDeath.state === 'interrupted' ? byDeath : replacedVerdictFor(item) + } // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` const settlementId = `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${generation}` // Calls an earlier settle closed with no proof, revised once a proof names their owner. - const mutations = provenUnverifiedToolCallRevisions(items, input.deathEvidence, journal) + const mutations = provenUnverifiedToolCallRevisions(items, input.deathEvidence, journal, replaced) for (const item of items) { const identity = parseAgentJournalItemKey(item.itemId) // A turn already settled (a person's Stop) ends its calls as it ended; only a turn still running @@ -219,34 +237,32 @@ export async function settleStaleStructuredAgentSessionState(input: { }) } } - const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal) + const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal, replaced) const turnEnds = [ ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), ...proven ] mutations.push(...turnEnds) - const evidence = input.deathEvidence - if ( - evidence && - (proven.length > 0 || - items.some( - (item) => - isInProgressStructuredAgentSessionItem(item) && verdictFor(item).state === 'interrupted' - )) && - !endedByPersonsStop(journal, turnEnds) - ) { - mutations.unshift({ - kind: 'item', - // Named by the death it explains, so a retry after a partly written settle adds no second row. - identity: { - provider: 'orca', - clientMessageId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` - }, - // The death evidence is Orca's log text, never a sentence for a person: the row says only - // that the provider stopped, and how Orca ended when the provider died with it. - body: orcaStopRowBody(input.failureTextContext, evidence.runtimeEnd), - turnScope: settledRootTurnScope(items, turnEnds) - }) + const stopRow = staleStopRow(input, items, (item): StaleCut => { + const unverifiable = isUnverifiableTurn(item) + if (!unverifiable && !isInProgressStructuredAgentSessionItem(item)) { + return null + } + const fence = journal.itemFence(item.itemId) + // An earlier settle's `unverifiable` turn is revised only by a death naming its owner. + if ( + unverifiable + ? fence !== undefined && fence === input.deathEvidence?.ownerFence + : deathVerdictFor(item).state === 'interrupted' + ) { + return { by: 'death' } + } + return fence !== undefined && replacedVerdictFor(item).state === 'interrupted' + ? { by: 'replaced', fence } + : null + }) + if (stopRow && !endedByPersonsStop(journal, turnEnds)) { + mutations.unshift({ ...stopRow, turnScope: settledRootTurnScope(items, turnEnds) }) } for (const chunk of partitionJournalLifecycleMutations(settlementId, mutations)) { await journal.appendLifecycleBatch({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index b6c8444a28bc..87c03c071ea9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -35,7 +35,11 @@ describe('StructuredAgentSessionReadableRestorer', () => { ) const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: { - store: { getRecord: () => null, listRecords: () => records }, + store: { + getRecord: () => null, + listRecords: () => records, + replacedRuntime: () => undefined + }, journalDatabase: openTestJournalHostDatabase(stateDirectory), logger: recordingStructuredAgentSessionLogger().logger }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts new file mode 100644 index 000000000000..4d4423fc22d4 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts @@ -0,0 +1,224 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalRenderItem +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + completedStructuredAgentTurnSeconds, + selectStructuredAgentTurnTimings +} from '../../../shared/structured-agent-session-turn-timing' +import { withNativeChatCutTurnNotices } from '../../../shared/native-chat-cut-turn-notice' +import { beginAgentSessionRuntimeIncarnationForTest } from '../../runtime/agent-session-runtime-attribution' +import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { HOST_TEST_NOW, hostTestAttachParams } from './structured-agent-session-host-test-data' +import { + createRestTestRig, + REST_TEST_CALLER, + restTestSend, + REST_TEST_SESSION as SESSION, + REST_TEST_THREAD as THREAD, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 +const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 +const CUT_REPLY = 'chunks 1-11' +const TURN = { provider: 'codex' as const, threadId: THREAD, turnId: 'cut-turn', ordinal: 0 } +const REPLY = { ...TURN, ordinal: 1 } +const NEXT_TURN = { provider: 'codex' as const, threadId: THREAD, turnId: 'next-turn', ordinal: 2 } +const NEXT_REPLY = { ...NEXT_TURN, ordinal: 3 } +// The provider's identity cannot be proven either way: the recovery releases it with no proof. +const UNPROVABLE: AgentSessionOwnerProbe = { outcome: 'indeterminate', reason: 'probe unavailable' } + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig({ idleSweep: { intervalMs: 3_600_000 } }) + vi.spyOn(Date, 'now').mockImplementation(() => rig.clock.now) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await rig.dispose() + closeTestJournalHostDatabases() +}) + +async function streamThenServerKilled(): Promise { + const attached = await rig.host.attach(REST_TEST_CALLER, hostTestAttachParams(null)) + if (!attached.ok) { + throw new Error(`attach refused: ${attached.refusal.code}`) + } + const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('long stream', attached.fence)) + if (!sent.ok) { + throw new Error(`send refused: ${sent.refusal.code}`) + } + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) + const events = rig.adapter.acquire.mock.calls[0]?.[0].events + if (!events) { + throw new Error('the provider has no event sink') + } + events.appendItem( + TURN, + { kind: 'turn', turnId: TURN.turnId, state: 'running', startedAt: HOST_TEST_NOW }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE, observedAt: HOST_TEST_NOW } + ) + for (let chunk = 1; chunk <= 11; chunk += 1) { + rig.clock.now = HOST_TEST_NOW + chunk * 500 + events.appendItem( + REPLY, + { + kind: 'message', + role: 'assistant', + state: 'running', + blocks: [{ type: 'text', text: `chunks 1-${chunk}` }] + }, + { + turnScope: { kind: 'turn', turnItemId: agentJournalItemKey(TURN) }, + observedAt: HOST_TEST_NOW + chunk * 500 + } + ) + await rig.host.flushStreamedEvents(SESSION) + } +} + +/** The server dies with no exit seen and no settle; a server process starts on the same host. */ +async function startServer(options: { newProcess: boolean }): Promise { + const old = rig.host + old.stopDelivery() + const { lifetime, runtimeState, sessions } = old.collaboratorsForTests() + lifetime.idleSweep.dispose() + await runtimeState.stopLeaseRenewal() + runtimeState.currentEventSink(SESSION)?.close() + await sessions.get(SESSION)?.journal.close() + closeTestJournalHostDatabases() + if (options.newProcess) { + beginAgentSessionRuntimeIncarnationForTest() + } + rig.clock.now = RESTARTED_AT + rig.store = await openTestAgentSessionRecordStore(rig.root) + rig.host = new StructuredAgentSessionHost({ + ...old.deps, + store: rig.store, + journalDatabase: openTestJournalHostDatabase(rig.root), + probeOwner: async () => UNPROVABLE, + now: () => rig.clock.now + }) +} + +function turnsOf(items: readonly AgentJournalRenderItem[]) { + return items.flatMap((item) => readAgentJournalTurn(item.body) ?? []) +} + +function repliesOf(items: readonly AgentJournalRenderItem[]): string[] { + return items.flatMap((item) => + item.body.kind === 'message' && item.body.role === 'assistant' + ? item.body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])) + : [] + ) +} + +function stopNotices(items: readonly AgentJournalRenderItem[]): AgentJournalRenderItem[] { + return withNativeChatCutTurnNotices(items, { agentName: 'Codex' }).filter( + (item) => item.body.kind === 'status' + ) +} + +async function expectCutTurnSettledOnce(): Promise { + const { items } = await rig.host.journalSnapshot(SESSION) + const cut = turnsOf(items).find((turn) => turn.turnId === TURN.turnId) + expect(cut).toMatchObject({ state: 'interrupted', completedAt: LAST_CHUNK_AT }) + const [timing] = selectStructuredAgentTurnTimings(items).values() + expect(completedStructuredAgentTurnSeconds(timing)).toBe(5) + expect(repliesOf(items).filter((text) => text === CUT_REPLY)).toHaveLength(1) + expect(stopNotices(items)).toHaveLength(1) +} + +describe('a server replaced mid-turn on the same execution host', () => { + it('interrupts the cut turn at its last saved output with no provider identity left', async () => { + await streamThenServerKilled() + await startServer({ newProcess: true }) + await rig.host.restoreReadableSessions() + await expectCutTurnSettledOnce() + + await rig.host.reconcileRestartLeases() + await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) + // Nothing proved the provider gone: the lease keeps no death and no identity. + const lease = rig.store.getRecord(SESSION)?.lease + expect(lease).toMatchObject({ claimStatus: 'released', deathEvidence: null }) + expect(lease?.ownerProcess).toBeNull() + await expectCutTurnSettledOnce() + + const attached = await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(lease?.runtimeFence ?? null) + ) + if (!attached.ok) { + throw new Error(`re-attach refused: ${attached.refusal.code}`) + } + const sent = await rig.host.send( + REST_TEST_CALLER, + restTestSend('after restart', attached.fence) + ) + expect(sent.ok).toBe(true) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2)) + const events = rig.adapter.acquire.mock.calls[1]?.[0].events + if (!events) { + throw new Error('the new provider has no event sink') + } + events.appendItem( + NEXT_TURN, + { kind: 'turn', turnId: NEXT_TURN.turnId, state: 'running', startedAt: RESTARTED_AT }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE, observedAt: RESTARTED_AT } + ) + events.appendItem( + NEXT_REPLY, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'fresh reply' }] }, + { turnScope: { kind: 'turn', turnItemId: agentJournalItemKey(NEXT_TURN) } } + ) + events.appendItem( + NEXT_TURN, + { + kind: 'turn', + turnId: NEXT_TURN.turnId, + state: 'completed', + startedAt: RESTARTED_AT, + completedAt: RESTARTED_AT + 1_000 + }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + await rig.host.flushStreamedEvents(SESSION) + + await expectCutTurnSettledOnce() + const { items } = await rig.host.journalSnapshot(SESSION) + expect(turnsOf(items).map((turn) => turn.state)).toEqual(['interrupted', 'completed']) + expect(repliesOf(items)).toEqual([CUT_REPLY, 'fresh reply']) + }) + + it('revises a turn an earlier start left unverifiable once a new server replaces its owner', async () => { + await streamThenServerKilled() + // The same process reopening its store replaced no runtime: nothing proves the turn over. + await startServer({ newProcess: false }) + await rig.host.restoreReadableSessions() + await rig.host.reconcileRestartLeases() + await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) + const left = turnsOf((await rig.host.journalSnapshot(SESSION)).items) + expect(left).toEqual([expect.objectContaining({ state: 'unverifiable' })]) + + await startServer({ newProcess: true }) + await rig.host.restoreReadableSessions() + await expectCutTurnSettledOnce() + + const settled = await rig.host.journalSnapshot(SESSION) + await startServer({ newProcess: true }) + await rig.host.restoreReadableSessions() + expect((await rig.host.journalSnapshot(SESSION)).items).toEqual(settled.items) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 7df838d04e82..1b74a23911f3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -28,7 +28,7 @@ afterAll(() => { }) const NO_OPEN_DEPS = { - store: { getRecord: () => null, listRecords: () => [] }, + store: { getRecord: () => null, listRecords: () => [], replacedRuntime: () => undefined }, journalDatabase: openTestJournalHostDatabase(stateDirectory), logger: recordingStructuredAgentSessionLogger().logger } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts index 616fce6a3a68..5deb2e482db9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts @@ -195,7 +195,8 @@ describe('a provider stream cut short before its next lease renewal', () => { expect(rig.adapter.closeSession).toHaveBeenCalledOnce() }) - it('leaves the end unverifiable when the execution host cannot prove owner death', async () => { + // A new server replacing the owner's runtime ends it anyway (structured-agent-session-replaced-server-turn). + it('leaves the end unverifiable when its own runtime reopens and cannot prove owner death', async () => { await streamBeforeCrash() await restartAfterCrash() rig.host.deps.probeOwner = async () => ({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-stop-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-stop-row.ts new file mode 100644 index 000000000000..ad39aa2975b0 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-stop-row.ts @@ -0,0 +1,68 @@ +// The row a stale settle writes about why the turns it cut stopped: a death that names their owner, +// or the replacement of the Orca runtime that held them. One row per proof, named by it, so a retry +// after a partly written settle adds no second row. + +import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' +import type { AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words' +import type { + AgentJournalRenderItem, + AgentJournalStatusItem +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import type { AgentSessionReplacedRuntime } from '../../runtime/agent-session-replaced-runtime' +import { orcaStopRowBody } from './structured-agent-session-orca-stop-row' + +/** Which proof ends an item the settle found running or left `unverifiable`, if any does. */ +export type StaleCut = { by: 'death' } | { by: 'replaced'; fence: number } | null + +export function isUnverifiableTurn(item: AgentJournalRenderItem): boolean { + return readAgentJournalTurn(item.body)?.state === 'unverifiable' +} + +export function staleStopRow( + input: { + sessionId: string + deathEvidence: AgentSessionDeathEvidence | null + replaced?: AgentSessionReplacedRuntime + failureTextContext?: AgentSessionFailureWordsContext + }, + items: readonly AgentJournalRenderItem[], + cutBy: (item: AgentJournalRenderItem) => StaleCut +): { + kind: 'item' + identity: { provider: 'orca'; clientMessageId: string } + body: AgentJournalStatusItem +} | null { + const cuts = items.map(cutBy) + const evidence = input.deathEvidence + // The death evidence is Orca's log text, never a sentence for a person: the row says only that + // the provider stopped, and how Orca ended when the provider died with it. + if (evidence && cuts.some((cut) => cut?.by === 'death')) { + return { + kind: 'item', + identity: { + provider: 'orca', + clientMessageId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` + }, + body: orcaStopRowBody(input.failureTextContext, evidence.runtimeEnd) + } + } + const fences = cuts.flatMap((cut) => (cut?.by === 'replaced' ? [cut.fence] : [])) + if (!input.replaced || fences.length === 0) { + return null + } + const fence = Math.max(...fences) + return { + kind: 'item', + // Named by the owner it explains, which every later runtime finds the same. + identity: { + provider: 'orca', + clientMessageId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:replaced-${fence}` + }, + body: orcaStopRowBody( + input.failureTextContext, + fence === input.replaced.fence ? input.replaced.runtimeEnd : undefined + ) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts index aa82c1c77941..1212546aa822 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.ts @@ -1,9 +1,10 @@ // What the host may durably say about a turn whose provider child is gone. // -// `interrupted` requires proof that the child which wrote the turn is gone: death evidence naming -// that turn's owner by fence — a watched exit, or a local probe that found the recorded pid gone or -// reused. A release nothing proved — lost contact, an unverifiable identity, a stop that outlived the -// ladder — carries none, and neither does a later owner's death; the turn is then `unverifiable` +// `interrupted` requires proof that the turn is over: death evidence naming that turn's owner by +// fence (a watched exit, or a local probe that found the recorded pid gone or reused), or the +// replacement of the Orca runtime that held the owner's pipes, which this host proves by holding its +// lock. A release nothing proved within one runtime (an unverifiable identity, a stop that outlived +// the ladder) carries none, and neither does a later owner's death; the turn is then `unverifiable` // with no end at all, until a proof naming its owner is written and revises it. import { @@ -21,6 +22,7 @@ import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import type { AgentSessionReplacedRuntime } from '../../runtime/agent-session-replaced-runtime' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' @@ -61,6 +63,20 @@ export function turnVerdictFromDeathEvidence( return { state: 'interrupted', completedAt: Math.min(lastAlive, evidence.observedAt) } } +/** A turn whose owner a replaced runtime held is over: nothing it did after reaches the chat. It + * ends at the owner's last proof of life, which that runtime wrote before this one took over. */ +export function turnVerdictFromReplacedRuntime( + replaced: AgentSessionReplacedRuntime | undefined, + turnFence: number | undefined, + liveAt?: number +): StructuredAgentSessionTurnVerdict { + if (!replaced || turnFence === undefined || turnFence > replaced.fence) { + return UNVERIFIABLE_TURN_VERDICT + } + const renewedAt = turnFence === replaced.fence ? (replaced.lastProvenAliveAt ?? 0) : 0 + return { state: 'interrupted', completedAt: Math.max(renewedAt, liveAt ?? 0) } +} + /** The latest saved output from this turn's owner, including a Stop that found it running. */ export function lastProvenTurnLiveAt( journal: Pick, @@ -109,21 +125,22 @@ export function runningTurnLifecycleRevisions( export function provenUnverifiableTurnRevisions( items: readonly AgentJournalRenderItem[], evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick + journal: Pick, + /** A runtime replacement proves it for every owner that runtime held, a death only for its own. */ + replaced?: AgentSessionReplacedRuntime ): JournalLifecycleMutationInput[] { - const ownerFence = evidence?.ownerFence - if (ownerFence === undefined) { - return [] - } return items.flatMap((item) => { const turn = readAgentJournalTurn(item.body) - return turn?.state === 'unverifiable' && journal.itemFence(item.itemId) === ownerFence - ? turnLifecycleRevision( - item, - turn, - turnVerdictFromDeathEvidence(evidence, ownerFence, lastProvenTurnLiveAt(journal, item)) - ) - : [] + if (turn?.state !== 'unverifiable') { + return [] + } + const fence = journal.itemFence(item.itemId) + const liveAt = lastProvenTurnLiveAt(journal, item) + const verdict = + evidence?.ownerFence !== undefined && fence === evidence.ownerFence + ? turnVerdictFromDeathEvidence(evidence, fence, liveAt) + : turnVerdictFromReplacedRuntime(replaced, fence, liveAt) + return verdict.state === 'interrupted' ? turnLifecycleRevision(item, turn, verdict) : [] }) } @@ -158,18 +175,19 @@ export function watchedExitRevisions( export function provenUnverifiedToolCallRevisions( items: readonly AgentJournalRenderItem[], evidence: AgentSessionDeathEvidence | null | undefined, - journal: Pick + journal: Pick, + replaced?: AgentSessionReplacedRuntime ): JournalLifecycleMutationInput[] { const ownerFence = evidence?.ownerFence - if (ownerFence === undefined) { - return [] - } + const proven = (fence: number | undefined) => + fence !== undefined && + (fence === ownerFence || (replaced !== undefined && fence <= replaced.fence)) return items.flatMap((item): JournalLifecycleMutationInput[] => { const identity = parseAgentJournalItemKey(item.itemId) return identity && item.body.kind === 'tool-call' && isUnverifiedEndAgentJournalToolCall(item.body) && - journal.itemFence(item.itemId) === ownerFence + proven(journal.itemFence(item.itemId)) ? [ { kind: 'item', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 768b6724caa2..bb8da5e3319e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -541,9 +541,10 @@ describe('already-wedged profiles become usable on load', () => { } ) - it('marks a running turn left behind by a released lease unverifiable on a cold acquire', async () => { + it('interrupts a running turn left behind by a released lease on a cold acquire', async () => { // No settlement latch: the record was released cleanly, but the journal still says a turn is - // running. The child that wrote it is gone and nothing observed its exit. + // running. Nothing observed the child's exit, but the runtime that held it was replaced, and + // nothing after its start proves it alive. await seedStore(wedgedRecord({ claimStatus: 'released', handoffStage: null })) await seedRunningTurn() openHost() @@ -553,8 +554,9 @@ describe('already-wedged profiles become usable on load', () => { expect(acquire).toHaveBeenCalledOnce() expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', - state: 'unverifiable', + state: 'interrupted', startedAt: NOW - 5_000, + completedAt: NOW - 5_000, recovered: true }) expect( diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index c78a433415dc..8d61b1bb6cfe 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -102,7 +102,7 @@ export class AgentSessionRecordStore { hostId: string }): AgentSessionRecordStore { const rows = loadAgentSessionStoreRows(args.journalDatabase.db) - const transactions = new AgentSessionStoreTransactions(args.journalDatabase, rows) + const transactions = new AgentSessionStoreTransactions(args.journalDatabase, rows, args.hostId) return new AgentSessionRecordStore(transactions, args.hostId) } @@ -120,6 +120,9 @@ export class AgentSessionRecordStore { listRecords = (): AgentSessionRecord[] => [...this.state.records.values()] + /** The runtimes this one replaced for a chat, as found at load; absent for a chat it began. */ + replacedRuntime = (sessionId: string) => this.transactions.replacedRuntimes.get(sessionId) + /** Every chat this host holds a row for, readable or not. */ listHeldSessionIds = (): string[] => heldAgentSessionIds(this.state) @@ -185,9 +188,7 @@ export class AgentSessionRecordStore { ) /** A record this build cannot validate: readable as present, never grantable as a writer. */ - isSessionUnreadable(sessionId: string): boolean { - return this.state.unreadableRecords.has(sessionId) - } + isSessionUnreadable = (sessionId: string): boolean => this.state.unreadableRecords.has(sessionId) listOperationRows = (): AgentSessionOperationRow[] => [...this.state.operations.values()] diff --git a/src/main/runtime/agent-session-replaced-runtime.test.ts b/src/main/runtime/agent-session-replaced-runtime.test.ts new file mode 100644 index 000000000000..afccc5b4b4be --- /dev/null +++ b/src/main/runtime/agent-session-replaced-runtime.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' +import type { AgentSessionLease } from '../../shared/agent-session-record' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../shared/agent-session-record.test-fixture' +import { agentSessionReplacedRuntimes } from './agent-session-replaced-runtime' +import type { AgentSessionStoreState } from './agent-session-store-state' + +const SESSION = 'session-alpha-1' +const OWNER = { hostId: 'local', pid: 4242, processStartTimeMs: 1, spawnToken: 'spawn-owner' } + +function loaded(lease: Partial, runtimeEnd?: 'crash'): AgentSessionStoreState { + const record = agentSessionRecordFixture( + agentSessionLeaseFixture({ unreconciled: true, ...lease }) + ) + return { + records: new Map([[SESSION, record]]), + operations: new Map(), + retiredClaimKeys: [], + unreadableRecords: new Map(), + sessionTabs: null, + runtimeEnds: runtimeEnd ? new Map([['earlier', runtimeEnd]]) : new Map() + } +} + +function replacedFor(state: AgentSessionStoreState) { + return agentSessionReplacedRuntimes(state, { + hostId: 'local', + incarnation: 'this-runtime' + }).get(SESSION) +} + +describe('the runtimes a new server replaced, as it loads their records', () => { + it('names every fence an earlier runtime granted, its owner renewal, and how it ended', () => { + expect( + replacedFor(loaded({ ownerProcess: { ...OWNER, runtime: 'earlier' } }, 'crash')) + ).toEqual({ fence: 7, lastProvenAliveAt: 30_000, runtimeEnd: 'crash' }) + }) + + it('covers a chat an earlier runtime released, which no owner holds now', () => { + expect( + replacedFor(loaded({ ownerProcess: null, reservedSpawnToken: null, claimStatus: 'released' })) + ).toEqual({ fence: 7 }) + }) + + it.each([ + ['an owner this runtime recorded', { ownerProcess: { ...OWNER, runtime: 'this-runtime' } }], + ['an owner on another host', { ownerProcess: { ...OWNER, hostId: 'remote' } }], + ['a terminal agent an older build recorded', { claimStatus: 'conflicted' as const }] + ])('replaces nothing for %s', (_label, lease) => { + expect(replacedFor(loaded(lease))).toBeUndefined() + }) +}) diff --git a/src/main/runtime/agent-session-replaced-runtime.ts b/src/main/runtime/agent-session-replaced-runtime.ts new file mode 100644 index 000000000000..d58eec7a13f2 --- /dev/null +++ b/src/main/runtime/agent-session-replaced-runtime.ts @@ -0,0 +1,43 @@ +// What this runtime took over from the Orca runtimes that ran chats on this state directory before +// it. It loaded their records holding the profile's instance lock, so none of them is still running: +// every provider they started lost its pipes with them, and nothing it did after can reach the chat. +// That is the execution host's own proof that a turn those runtimes left running is over, whether or +// not the provider process itself can still be proven gone. + +import type { AgentSessionOrcaStopCause } from '../../shared/agent-session-orca-stop' +import type { AgentSessionStoreState } from './agent-session-store-state' + +export type AgentSessionReplacedRuntime = { + /** The highest fence a replaced runtime granted; every lower fence was granted before it. */ + fence: number + /** The last renewal of the owner loaded at `fence`, its last proof of life. */ + lastProvenAliveAt?: number + /** How the runtime holding the owner loaded at `fence` ended, when its record says. */ + runtimeEnd?: AgentSessionOrcaStopCause +} + +/** One entry per chat whose fences were all granted by runtimes this one replaced. */ +export function agentSessionReplacedRuntimes( + state: AgentSessionStoreState, + args: { hostId: string; incarnation: string } +): ReadonlyMap { + const replaced = new Map() + for (const [sessionId, { lease }] of state.records) { + const owner = lease.ownerProcess + // A terminal's own agent never wrote through Orca's pipes; another host's runtime is not one this + // replaced; and an owner this runtime recorded (a host reinstalled in one process) may still be held. + if ( + lease.claimStatus === 'conflicted' || + (owner && (owner.hostId !== args.hostId || owner.runtime === args.incarnation)) + ) { + continue + } + const runtimeEnd = owner?.runtime ? state.runtimeEnds?.get(owner.runtime) : undefined + replaced.set(sessionId, { + fence: lease.runtimeFence, + ...(owner ? { lastProvenAliveAt: lease.lastRenewedAt } : {}), + ...(runtimeEnd ? { runtimeEnd } : {}) + }) + } + return replaced +} diff --git a/src/main/runtime/agent-session-store-transactions.ts b/src/main/runtime/agent-session-store-transactions.ts index 3877da33af42..e70875904c1b 100644 --- a/src/main/runtime/agent-session-store-transactions.ts +++ b/src/main/runtime/agent-session-store-transactions.ts @@ -21,6 +21,10 @@ import { beginAgentSessionRuntimeRecord, readAgentSessionRuntimeEnds } from './agent-session-runtime-end-record' +import { + agentSessionReplacedRuntimes, + type AgentSessionReplacedRuntime +} from './agent-session-replaced-runtime' import { agentSessionStoreDraftRowWrites, draftAgentSessionStoreState, @@ -81,16 +85,23 @@ type StagedStoreTransaction = { export class AgentSessionStoreTransactions { private queue: Promise = Promise.resolve() private published: AgentSessionStoreState + /** What this runtime took over from the runtimes before it, as loaded; never changes after. */ + readonly replacedRuntimes: ReadonlyMap constructor( private readonly journalDatabase: JournalHostDatabase, - loaded: AgentSessionStoreState + loaded: AgentSessionStoreState, + hostId: string ) { freezeRows(loaded, null) this.published = { ...loaded, runtimeEnds: readAgentSessionRuntimeEnds(journalDatabase.stateDirectory) } + this.replacedRuntimes = agentSessionReplacedRuntimes(this.published, { + hostId, + incarnation: agentSessionRuntimeIncarnation() + }) // Before any owner this runtime records: a crash is concluded only from a recorded start. beginAgentSessionRuntimeRecord( journalDatabase.stateDirectory, From 414c424c5afb03ffa630cbe2184dcf4df4852a8e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:22:34 -0700 Subject: [PATCH 09/13] fix(native-chat): require the record store for any journal open that settles Only an acquisition's open may skip it; a reader open without the store would leave a replaced server's turns unverifiable. --- ...tructured-agent-session-conversation-open.ts | 15 +++++++++++++-- ...d-agent-session-replaced-server-turn.test.ts | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index 3dbe79ed697a..85738139603c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -79,9 +79,20 @@ export async function openStructuredAgentSessionConversation( return opened.session } -/** The open itself, indexed by nobody yet: the caller adopts the result. */ +/** The open itself, indexed by nobody yet: the caller adopts the result. Any open that settles + * needs the store, which names the runtimes this one replaced. */ +export async function openStructuredAgentSessionConversationJournal( + deps: StructuredAgentSessionConversationOpenDeps, + record: AgentSessionRecord, + options?: StructuredAgentSessionConversationOpenOptions +): Promise +/** An acquisition's open settles nothing itself, so it needs no store. */ +export async function openStructuredAgentSessionConversationJournal( + deps: Omit, + record: AgentSessionRecord, + options: { acquisition: true } +): Promise export async function openStructuredAgentSessionConversationJournal( - /** An acquisition's open settles nothing itself, so it needs no store. */ deps: Omit & Partial>, record: AgentSessionRecord, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts index 4d4423fc22d4..822fb635b966 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts @@ -5,6 +5,7 @@ import { type AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { completedStructuredAgentTurnSeconds, @@ -17,6 +18,10 @@ import { closeTestJournalHostDatabases, openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + openStructuredAgentSessionConversationJournal, + type StructuredAgentSessionConversationOpenDeps +} from './structured-agent-session-conversation-open' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { HOST_TEST_NOW, hostTestAttachParams } from './structured-agent-session-host-test-data' import { @@ -221,4 +226,16 @@ describe('a server replaced mid-turn on the same execution host', () => { await rig.host.restoreReadableSessions() expect((await rig.host.journalSnapshot(SESSION)).items).toEqual(settled.items) }) + + it('only lets an acquisition open a journal without the store', () => { + const open = ( + deps: Omit, + record: AgentSessionRecord + ) => { + void openStructuredAgentSessionConversationJournal(deps, record, { acquisition: true }) + // @ts-expect-error a reader open settles, so without the store its turns would stay unverifiable + void openStructuredAgentSessionConversationJournal(deps, record) + } + expect(open).toBeTypeOf('function') + }) }) From a6fd5c667b88434ee45c62451ec9f2eaba9f685e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:43:58 -0700 Subject: [PATCH 10/13] fix(native-chat): say a remote host's Orca restarted, not that it was closed A reply cut off when a remote host's Orca server stopped read "Orca on was closed while this response was in progress." Nobody closed anything there: the server stopped and Orca started it again before the row could show. For a chat on a remote environment, cause quit now reads "Orca on restarted while this response was in progress." The desktop's own quit keeps "was closed". Renderer-only; no wire change. --- .../NativeChatInterruptedContinue.test.tsx | 13 +++++-- .../NativeChatInterruptedContinue.tsx | 6 +++- .../native-chat/NativeChatNoticeRow.test.tsx | 34 ++++++++++++++++--- .../native-chat/NativeChatNoticeRow.tsx | 4 +-- .../native-chat-orca-stop-context.ts | 7 ++-- .../native-chat-orca-stop-words.ts | 23 +++++++++++-- src/renderer/src/i18n/locales/en.json | 1 + src/renderer/src/i18n/locales/es.json | 1 + src/renderer/src/i18n/locales/fr.json | 1 + src/renderer/src/i18n/locales/ja.json | 1 + src/renderer/src/i18n/locales/ko.json | 1 + src/renderer/src/i18n/locales/zh.json | 1 + 12 files changed, 78 insertions(+), 15 deletions(-) diff --git a/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.test.tsx b/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.test.tsx index de80b08e4c0b..6c35294a4ffe 100644 --- a/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.test.tsx @@ -89,9 +89,8 @@ function Harness(props: Props): React.JSX.Element { {continuation.offeredTurnItemId ?? 'none'} {String(continuation.view.continueAvailable)} - - {composerError ?? continuation.continueError?.text ?? 'none'} - + {String(continuation.view.remoteHost)} + {composerError ?? continuation.continueError?.text ?? 'none'} @@ -131,6 +130,14 @@ describe('Continue on a reply an Orca stop cut off', () => { expect(screen.getByTestId('offered')).toHaveTextContent('none') }) + it("tells the chat's rows whether its host is remote", () => { + render() + expect(screen.getByTestId('remote')).toHaveTextContent('true') + cleanup() + render() + expect(screen.getByTestId('remote')).toHaveTextContent('false') + }) + it('says what it does, for a reader and on hover', () => { render() expect(continueButton()).toHaveAccessibleDescription( diff --git a/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx b/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx index 2e3e0be96de3..0c3b64dda016 100644 --- a/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx +++ b/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx @@ -87,7 +87,11 @@ export function useNativeChatInterruptedContinuation(input: { // Unknown counts as able: a host that writes cause rows has Continue, and the words stay put. const continueAvailable = capability !== 'unsupported' // One object per change, so the chat's rows re-render only when what they show changes. - const view = useMemo(() => ({ hostLabel, continueAvailable }), [hostLabel, continueAvailable]) + const remoteHost = target.kind === 'environment' + const view = useMemo( + () => ({ hostLabel, remoteHost, continueAvailable }), + [hostLabel, remoteHost, continueAvailable] + ) const failedHere = failedOn !== null && cut?.turnItemId === failedOn ? translate( diff --git a/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx b/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx index db62bc280a7e..90fd6445cd82 100644 --- a/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx @@ -14,15 +14,17 @@ afterEach(cleanup) function orcaStopView( hostLabel: string | null, - continueAvailable: boolean + continueAvailable: boolean, + remoteHost: boolean ): NativeChatOrcaStopView { - return { hostLabel, continueAvailable } + return { hostLabel, remoteHost, continueAvailable } } function renderStatus( body: AgentJournalStatusItem, hostLabel: string | null = null, - continueAvailable = false + continueAvailable = false, + remoteHost = false ) { const [message] = projectStructuredItemsToNativeChat([ { @@ -34,7 +36,7 @@ function renderStatus( turnScope: { kind: 'turn', turnItemId: 'cut-turn' } } ]) - const view = orcaStopView(hostLabel, continueAvailable) + const view = orcaStopView(hostLabel, continueAvailable, remoteHost) return render( @@ -76,6 +78,30 @@ describe('the row an Orca stop leaves', () => { ).toBeInTheDocument() }) + it("says a remote host's Orca restarted, not that it was closed", () => { + renderStatus(orcaStopRow('quit'), 'QA SSH', true, true) + expect( + screen.getByText('Orca on QA SSH restarted while this response was in progress.') + .parentElement?.parentElement + ).toHaveClass('text-muted-foreground') + expect(screen.queryByText(/was closed/)).toBeNull() + }) + + it("keeps this desktop's quit as closed", () => { + renderStatus(orcaStopRow('quit'), 'studio-mac', true, false) + expect( + screen.getByText('Orca on studio-mac was closed while this response was in progress.') + ).toBeInTheDocument() + }) + + it.each([ + ['update', 'Orca on QA SSH restarted for an update while this response was in progress.'], + ['crash', 'Orca on QA SSH stopped unexpectedly while this response was in progress.'] + ])("keeps a remote host's %s words", (cause, sentence) => { + renderStatus(orcaStopRow(cause), 'QA SSH', true, true) + expect(screen.getByText(sentence)).toBeInTheDocument() + }) + // A client that re-words unnamed host rows keeps this row's presentation and cause, neutral. it('stays neutral, naming the cause, as a reader that re-presented it neutral hands it on', () => { // Its words and tone replaced, and the failure fact the old words came from dropped with them. diff --git a/src/renderer/src/components/native-chat/NativeChatNoticeRow.tsx b/src/renderer/src/components/native-chat/NativeChatNoticeRow.tsx index 47856cdaae84..370e5988e1b7 100644 --- a/src/renderer/src/components/native-chat/NativeChatNoticeRow.tsx +++ b/src/renderer/src/components/native-chat/NativeChatNoticeRow.tsx @@ -101,10 +101,10 @@ export function NativeChatNoticeRow({ // The host's row about an Orca stop names the cause and the machine, muted: Orca stopped, not the // agent. With no machine to name it keeps the host's own words. const { orcaStop } = block - const { hostLabel, continueAvailable } = orcaStopView + const { hostLabel, remoteHost, continueAvailable } = orcaStopView const named = orcaStop !== undefined && hostLabel !== null const text = named - ? nativeChatOrcaStopRowText(orcaStop.cause, hostLabel, { continueAvailable }) + ? nativeChatOrcaStopRowText(orcaStop.cause, hostLabel, { continueAvailable, remoteHost }) : block.text const tone = named || block.presentation === AGENT_SESSION_ORCA_STOP_PRESENTATION ? 'notice' : block.tone diff --git a/src/renderer/src/components/native-chat/native-chat-orca-stop-context.ts b/src/renderer/src/components/native-chat/native-chat-orca-stop-context.ts index 216face98546..0b2f006af717 100644 --- a/src/renderer/src/components/native-chat/native-chat-orca-stop-context.ts +++ b/src/renderer/src/components/native-chat/native-chat-orca-stop-context.ts @@ -1,15 +1,18 @@ import { createContext, useContext } from 'react' /** What a structured chat tells its rows about an Orca stop: the name of the machine whose Orca runs - * it, as Orca shows that host everywhere (null when it has none to show), and whether its host can - * continue a cut, which only a host known not to has no Continue for. */ + * it, as Orca shows that host everywhere (null when it has none to show), whether that machine is a + * remote host rather than this desktop, and whether its host can continue a cut, which only a host + * known not to has no Continue for. */ export type NativeChatOrcaStopView = { hostLabel: string | null + remoteHost: boolean continueAvailable: boolean } export const NativeChatOrcaStopContext = createContext({ hostLabel: null, + remoteHost: false, continueAvailable: false }) diff --git a/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts b/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts index 46e43e964ba5..e876f78538bf 100644 --- a/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts +++ b/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts @@ -26,17 +26,34 @@ const STOPPED: Record string> = ) } +// A remote host's graceful stop is no one closing Orca, and it is back by the time this shows. +const remoteQuit = (machine: string): string => + translate( + 'components.native-chat.notices.orcaStopRemoteRestart', + 'Orca on {{machine}} restarted while this response was in progress.', + { machine } + ) + +function stoppedSentence( + cause: AgentSessionOrcaStopCause, + machine: string, + remoteHost: boolean +): string { + return cause === 'quit' && remoteHost ? remoteQuit(machine) : STOPPED[cause](machine) +} + /** The row's sentence. Where the host can continue a cut, Continue is the way on, so the row does * not say it again; decided by the host, not the button, so the words never change on screen. */ export function nativeChatOrcaStopRowText( cause: AgentSessionOrcaStopCause, machine: string, - options: { continueAvailable: boolean } + options: { continueAvailable: boolean; remoteHost: boolean } ): string { + const stopped = stoppedSentence(cause, machine, options.remoteHost) return options.continueAvailable - ? STOPPED[cause](machine) + ? stopped : joinSentences([ - STOPPED[cause](machine), + stopped, translate( 'components.native-chat.notices.orcaStopCanContinue', 'You can continue in this conversation.' diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 70ec790cffc6..35c68d006c83 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -18137,6 +18137,7 @@ "orcaStopUpdate": "Orca on {{machine}} restarted for an update while this response was in progress.", "orcaStopQuit": "Orca on {{machine}} was closed while this response was in progress.", "orcaStopCrash": "Orca on {{machine}} stopped unexpectedly while this response was in progress.", + "orcaStopRemoteRestart": "Orca on {{machine}} restarted while this response was in progress.", "orcaStopCanContinue": "You can continue in this conversation." }, "composer": { diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index fcc5d506162c..903a110f3b3c 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -18279,6 +18279,7 @@ "orcaStopUpdate": "Orca en {{machine}} se reinició para actualizarse mientras esta respuesta estaba en curso.", "orcaStopQuit": "Orca en {{machine}} se cerró mientras esta respuesta estaba en curso.", "orcaStopCrash": "Orca en {{machine}} se detuvo inesperadamente mientras esta respuesta estaba en curso.", + "orcaStopRemoteRestart": "Orca en {{machine}} se reinició mientras esta respuesta estaba en curso.", "orcaStopCanContinue": "Puedes continuar en esta conversación." }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index a566463c9b23..bc45039e2e48 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -18264,6 +18264,7 @@ "orcaStopUpdate": "Orca sur {{machine}} a redémarré pour une mise à jour pendant que cette réponse était en cours.", "orcaStopQuit": "Orca sur {{machine}} a été fermé pendant que cette réponse était en cours.", "orcaStopCrash": "Orca sur {{machine}} s'est arrêté de manière inattendue pendant que cette réponse était en cours.", + "orcaStopRemoteRestart": "Orca sur {{machine}} a redémarré pendant que cette réponse était en cours.", "orcaStopCanContinue": "Vous pouvez continuer dans cette conversation." }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index b05e33cc1c9a..2caa1538cb20 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -18198,6 +18198,7 @@ "orcaStopUpdate": "この応答の途中で、{{machine}} の Orca がアップデートのため再起動しました。", "orcaStopQuit": "この応答の途中で、{{machine}} の Orca が終了しました。", "orcaStopCrash": "この応答の途中で、{{machine}} の Orca が予期せず停止しました。", + "orcaStopRemoteRestart": "この応答の途中で、{{machine}} の Orca が再起動しました。", "orcaStopCanContinue": "この会話で続行できます。" }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 922180cbd59d..f57ea6c15287 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -18198,6 +18198,7 @@ "orcaStopUpdate": "이 응답이 진행되는 동안 {{machine}}의 Orca가 업데이트를 위해 다시 시작되었습니다.", "orcaStopQuit": "이 응답이 진행되는 동안 {{machine}}의 Orca가 종료되었습니다.", "orcaStopCrash": "이 응답이 진행되는 동안 {{machine}}의 Orca가 예기치 않게 중지되었습니다.", + "orcaStopRemoteRestart": "이 응답이 진행되는 동안 {{machine}}의 Orca가 다시 시작되었습니다.", "orcaStopCanContinue": "이 대화에서 계속할 수 있습니다." }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index d4f231058078..0b657496dc40 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -18163,6 +18163,7 @@ "orcaStopUpdate": "此回复进行期间，{{machine}} 上的 Orca 为更新而重新启动。", "orcaStopQuit": "此回复进行期间，{{machine}} 上的 Orca 已关闭。", "orcaStopCrash": "此回复进行期间，{{machine}} 上的 Orca 意外停止。", + "orcaStopRemoteRestart": "此回复进行期间，{{machine}} 上的 Orca 已重新启动。", "orcaStopCanContinue": "你可以在此对话中继续。" }, "providerFrame": { From cf91fbc715aa9ba0ef104ae8b2cec505d3d56f15 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:56:34 -0700 Subject: [PATCH 11/13] fix(native-chat): say a remote host's Orca stopped, true for a shared desktop too --- .../native-chat/NativeChatInterruptedContinue.tsx | 2 +- .../src/components/native-chat/NativeChatNoticeRow.test.tsx | 6 +++--- .../components/native-chat/native-chat-orca-stop-words.ts | 6 +++--- src/renderer/src/i18n/locales/en.json | 2 +- src/renderer/src/i18n/locales/es.json | 2 +- src/renderer/src/i18n/locales/fr.json | 2 +- src/renderer/src/i18n/locales/ja.json | 2 +- src/renderer/src/i18n/locales/ko.json | 2 +- src/renderer/src/i18n/locales/zh.json | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx b/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx index 0c3b64dda016..ae8a9a503adb 100644 --- a/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx +++ b/src/renderer/src/components/native-chat/NativeChatInterruptedContinue.tsx @@ -21,7 +21,7 @@ import type { NativeChatComposerNoticeContent } from './native-chat-composer-not type ContinueAnswer = { outcome?: string } export type NativeChatInterruptedContinuation = { - /** What the chat's rows are told: the machine's name and whether its host can continue a cut. */ + /** Told to the chat's rows: the machine's name, whether it is remote, and if it can continue. */ view: NativeChatOrcaStopView /** The cut turn Continue is offered on right now, if any. */ offeredTurnItemId: string | null diff --git a/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx b/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx index 90fd6445cd82..951b04e08959 100644 --- a/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatNoticeRow.test.tsx @@ -78,11 +78,11 @@ describe('the row an Orca stop leaves', () => { ).toBeInTheDocument() }) - it("says a remote host's Orca restarted, not that it was closed", () => { + it("says a remote host's Orca stopped, not that it was closed", () => { renderStatus(orcaStopRow('quit'), 'QA SSH', true, true) expect( - screen.getByText('Orca on QA SSH restarted while this response was in progress.') - .parentElement?.parentElement + screen.getByText('Orca on QA SSH stopped while this response was in progress.').parentElement + ?.parentElement ).toHaveClass('text-muted-foreground') expect(screen.queryByText(/was closed/)).toBeNull() }) diff --git a/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts b/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts index e876f78538bf..10df0deab3d6 100644 --- a/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts +++ b/src/renderer/src/components/native-chat/native-chat-orca-stop-words.ts @@ -26,11 +26,11 @@ const STOPPED: Record string> = ) } -// A remote host's graceful stop is no one closing Orca, and it is back by the time this shows. +// A remote graceful stop is a server stop or a shared desktop's quit; "stopped" is true of both. const remoteQuit = (machine: string): string => translate( - 'components.native-chat.notices.orcaStopRemoteRestart', - 'Orca on {{machine}} restarted while this response was in progress.', + 'components.native-chat.notices.orcaStopRemote', + 'Orca on {{machine}} stopped while this response was in progress.', { machine } ) diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 35c68d006c83..e59370876762 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -18137,7 +18137,7 @@ "orcaStopUpdate": "Orca on {{machine}} restarted for an update while this response was in progress.", "orcaStopQuit": "Orca on {{machine}} was closed while this response was in progress.", "orcaStopCrash": "Orca on {{machine}} stopped unexpectedly while this response was in progress.", - "orcaStopRemoteRestart": "Orca on {{machine}} restarted while this response was in progress.", + "orcaStopRemote": "Orca on {{machine}} stopped while this response was in progress.", "orcaStopCanContinue": "You can continue in this conversation." }, "composer": { diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 903a110f3b3c..0aca85902120 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -18279,7 +18279,7 @@ "orcaStopUpdate": "Orca en {{machine}} se reinició para actualizarse mientras esta respuesta estaba en curso.", "orcaStopQuit": "Orca en {{machine}} se cerró mientras esta respuesta estaba en curso.", "orcaStopCrash": "Orca en {{machine}} se detuvo inesperadamente mientras esta respuesta estaba en curso.", - "orcaStopRemoteRestart": "Orca en {{machine}} se reinició mientras esta respuesta estaba en curso.", + "orcaStopRemote": "Orca en {{machine}} se detuvo mientras esta respuesta estaba en curso.", "orcaStopCanContinue": "Puedes continuar en esta conversación." }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index bc45039e2e48..4a6fc19fb5e2 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -18264,7 +18264,7 @@ "orcaStopUpdate": "Orca sur {{machine}} a redémarré pour une mise à jour pendant que cette réponse était en cours.", "orcaStopQuit": "Orca sur {{machine}} a été fermé pendant que cette réponse était en cours.", "orcaStopCrash": "Orca sur {{machine}} s'est arrêté de manière inattendue pendant que cette réponse était en cours.", - "orcaStopRemoteRestart": "Orca sur {{machine}} a redémarré pendant que cette réponse était en cours.", + "orcaStopRemote": "Orca sur {{machine}} s'est arrêté pendant que cette réponse était en cours.", "orcaStopCanContinue": "Vous pouvez continuer dans cette conversation." }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 2caa1538cb20..3f44ce5f1dd0 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -18198,7 +18198,7 @@ "orcaStopUpdate": "この応答の途中で、{{machine}} の Orca がアップデートのため再起動しました。", "orcaStopQuit": "この応答の途中で、{{machine}} の Orca が終了しました。", "orcaStopCrash": "この応答の途中で、{{machine}} の Orca が予期せず停止しました。", - "orcaStopRemoteRestart": "この応答の途中で、{{machine}} の Orca が再起動しました。", + "orcaStopRemote": "この応答の途中で、{{machine}} の Orca が停止しました。", "orcaStopCanContinue": "この会話で続行できます。" }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index f57ea6c15287..5b80a1344f21 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -18198,7 +18198,7 @@ "orcaStopUpdate": "이 응답이 진행되는 동안 {{machine}}의 Orca가 업데이트를 위해 다시 시작되었습니다.", "orcaStopQuit": "이 응답이 진행되는 동안 {{machine}}의 Orca가 종료되었습니다.", "orcaStopCrash": "이 응답이 진행되는 동안 {{machine}}의 Orca가 예기치 않게 중지되었습니다.", - "orcaStopRemoteRestart": "이 응답이 진행되는 동안 {{machine}}의 Orca가 다시 시작되었습니다.", + "orcaStopRemote": "이 응답이 진행되는 동안 {{machine}}의 Orca가 중지되었습니다.", "orcaStopCanContinue": "이 대화에서 계속할 수 있습니다." }, "providerFrame": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 0b657496dc40..79014da68d55 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -18163,7 +18163,7 @@ "orcaStopUpdate": "此回复进行期间，{{machine}} 上的 Orca 为更新而重新启动。", "orcaStopQuit": "此回复进行期间，{{machine}} 上的 Orca 已关闭。", "orcaStopCrash": "此回复进行期间，{{machine}} 上的 Orca 意外停止。", - "orcaStopRemoteRestart": "此回复进行期间，{{machine}} 上的 Orca 已重新启动。", + "orcaStopRemote": "此回复进行期间，{{machine}} 上的 Orca 已停止。", "orcaStopCanContinue": "你可以在此对话中继续。" }, "providerFrame": { From 9c6a3ca7e86c9cb5ddcc9be87755aa78097f515d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:51:21 -0700 Subject: [PATCH 12/13] test(native-chat): register Grok without compaction in the recovery-output rig Main's #26502 made Grok declare compaction; the rest-test rig adapter has no compact, so the registry refused it before any recovery case ran. --- ...tructured-agent-session-restart-recovery-output.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts index 401e98d8ae63..1c78db236f3e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts @@ -42,8 +42,13 @@ beforeEach(async () => { if (!spec) { throw new Error('ACP agent definition absent') } + const definition = acpStructuredAgentDefinition(spec) + // The rig adapter has no compact; this recovery never compacts. rig.host.deps.agents = new StructuredAgentRegistry([ - { definition: acpStructuredAgentDefinition(spec), adapter: rig.host.deps.adapter } + { + definition: { ...definition, capabilities: { ...definition.capabilities, compact: false } }, + adapter: rig.host.deps.adapter + } ]) rig.adapter.acquire.mockImplementation(async ({ fence, spawnToken }) => ({ acquisitionGeneration: 'acp-generation', From 9997d44af7cb4f0eee134fd5c274e01882cc97fa Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:30:23 -0700 Subject: [PATCH 13/13] chore(native-chat): say a replaced runtime also revises an unverifiable turn --- .../structured-agent-session-dead-generation-settlement.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index c8fc1e626800..0a32aa14d6fd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -250,7 +250,7 @@ export async function settleStaleStructuredAgentSessionState(input: { return null } const fence = journal.itemFence(item.itemId) - // An earlier settle's `unverifiable` turn is revised only by a death naming its owner. + // An earlier settle's `unverifiable` turn is revised by a death naming its owner or by its replaced runtime. if ( unverifiable ? fence !== undefined && fence === input.deathEvidence?.ownerFence