From 98a2e27cda92f5ee6550b5872b67aec15a6a0d73 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:56:25 -0700 Subject: [PATCH 001/107] fix(native-chat): report a failed startup chat reconcile instead of failing app startup At startup the chat host re-checks every saved chat's lease and writes the result to agent-sessions.json. If that write failed (the file lock gave up, the file could not be written, or the file was written by a newer Orca and is read-only here), reconcileRestartLeases rejected, the startup IPC call rejected, and the renderer fell into its degraded "Session restore failed. Changes won't be saved until restart" mode. The reconcile is bookkeeping: a lease left unreconciled grants no writer, and every attach, send and read of a chat reconciles its own lease again. So the startup reconcile now reports its failure through a new optional host dependency, onStartupReconcileFailure, and resolves. The runtime routes it to its onError sink under the scope structured-agent-session-startup-reconcile, or logs it when no sink is installed (the desktop installs none). --- .../structured-agent-session-host-types.ts | 3 + .../structured-agent-session-host.ts | 13 +- ...uctured-agent-session-restart-reconcile.ts | 16 ++ ...-session-startup-reconcile-failure.test.ts | 145 ++++++++++++++++++ .../structured-agent-session-runtime.ts | 7 + ...ed-agent-session-startup-reconcile.test.ts | 110 +++++++++++++ 6 files changed, 287 insertions(+), 7 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts create mode 100644 src/main/runtime/structured-agent-session-startup-reconcile.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index 80ea85e86835..bb858f804838 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -120,6 +120,9 @@ export type StructuredAgentSessionHostDeps = { /** Whether an orchestration dispatch still owns this session's worker; absent answers no. */ hasOpenDispatch?: (record: AgentSessionRecord) => boolean onEventSinkError?: (input: { sessionId: string; error: unknown }) => void + /** A startup reconcile that refused or threw. Startup carries on: every attach, send and read + * reconciles its own chat again. */ + onStartupReconcileFailure?: (failure: unknown) => void /** Every status projection this host publishes. `replay` marks a re-projection of state the host * already knew (restore, an arriving subscriber) rather than a fresh journal edge. */ onSessionStatusChanged?: ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index efdd83d9f146..56f206db1a8d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -10,7 +10,10 @@ import type { AgentJournalSnapshot } from '../../../shared/agent-session-journal import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' import type * as SessionWire from '../../../shared/agent-session-wire' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { createRestartReconciler } from './structured-agent-session-restart-reconcile' +import { + createRestartReconciler, + reconcileLeasesAtStartup +} from './structured-agent-session-restart-reconcile' import type { AgentSessionSubscribeInput } from './structured-agent-session-subscribers' import { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' import * as providerSupport from './structured-agent-session-provider-support' @@ -228,12 +231,8 @@ export class StructuredAgentSessionHost { } } - reconcileRestartLeases = async (): Promise => { - const refusal = await this.reconcileLeases('startup') - if (refusal) { - throw new Error(refusal.code) - } - } + reconcileRestartLeases = (): Promise => + reconcileLeasesAtStartup(this.reconcileLeases, this.deps.onStartupReconcileFailure) restoreReadableSessions = (sessionIds?: readonly string[]): Promise => this.restore.restoreReadableSessions(sessionIds) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index 3b6cc30271e5..b4d1380090d5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -45,6 +45,22 @@ export function createRestartReconciler(deps: { } } +/** Startup never fails on this verdict: a lease left unreconciled grants no writer, and the next + * attach, send or read of that chat reconciles it again. So a failure is reported, never owed. */ +export async function reconcileLeasesAtStartup( + reconcile: (sessionId: string) => Promise, + onFailure: ((failure: unknown) => void) | undefined +): Promise { + try { + const refusal = await reconcile('startup') + if (refusal) { + onFailure?.(refusal) + } + } catch (error) { + onFailure?.(error) + } +} + async function reconcileCurrentLeases(deps: { store: AgentSessionRecordStore probe: (record: AgentSessionRecord) => Promise diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts new file mode 100644 index 000000000000..0e96294872bc --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts @@ -0,0 +1,145 @@ +// The startup reconcile is bookkeeping: a lock that gives up or a store this build may not write +// is reported, and startup carries on. Nothing is owed after it, because every chat reconciles its +// own lease again when it is sent to or read. + +import { cp, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import type * as FileTransactionLock from '../../file-transaction-lock' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + AGENT_SESSION_STORE_SCHEMA_VERSION, + agentSessionStorePath +} from '../../runtime/agent-session-record-store-file' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + adapter, + attach, + CALLER, + envelope, + hostTestState, + replaceHostTestState +} from './structured-agent-session-host-test-harness' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestMessage +} from './structured-agent-session-host-test-data' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' + +const lock = vi.hoisted(() => ({ failing: false })) + +vi.mock('../../file-transaction-lock', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + withFileTransactionLock: (...args: Parameters) => + lock.failing + ? // What proper-lockfile throws once its retries give up. + Promise.reject( + Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED' }) + ) + : actual.withFileTransactionLock(...args) + } +}) + +const relaunchedRoots: string[] = [] + +afterEach(async () => { + lock.failing = false + await Promise.all( + relaunchedRoots.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ) +}) + +/** A host with one chat, relaunched over a copy of its files; `rewrite` edits the copied store. */ +async function relaunch(rewrite?: (json: Record) => void) { + const dying = hostTestState() + await attach() + // An empty renewal queues behind every record write, so they are on disk. + await dying.store.renewLeases([]) + const relaunched = `${dying.root}-relaunched` + relaunchedRoots.push(relaunched) + // A dead process holds no lock. + await cp(dying.root, relaunched, { + recursive: true, + filter: (source) => !source.includes('.lock') + }) + const storeDirectory = join(relaunched, 'store') + if (rewrite) { + const path = agentSessionStorePath(storeDirectory) + const json = JSON.parse(await readFile(path, 'utf-8')) + rewrite(json) + await writeFile(path, JSON.stringify(json)) + } + const store = await AgentSessionRecordStore.open({ directory: storeDirectory, hostId: 'local' }) + const onStartupReconcileFailure = vi.fn() + const host = new StructuredAgentSessionHost({ + store, + adapter: adapter(), + journalDatabase: openTestJournalHostDatabase(relaunched), + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-next', + probeOwner: async () => ({ outcome: 'pid-absent' }), + now: () => NOW, + onStartupReconcileFailure + }) + replaceHostTestState({ store, host }) + return { host, store, storeDirectory, onStartupReconcileFailure } +} + +it('reports a startup reconcile whose store write fails, and does not reject', async () => { + const { host, store, onStartupReconcileFailure } = await relaunch() + + lock.failing = true + await expect(host.reconcileRestartLeases()).resolves.toBeUndefined() + + expect(onStartupReconcileFailure).toHaveBeenCalledOnce() + expect(onStartupReconcileFailure).toHaveBeenCalledWith( + expect.objectContaining({ code: 'ELOCKED' }) + ) + // Nothing was adjudicated, so the lease still grants no writer. + expect(store.getRecord(SESSION)?.lease.unreconciled).toBe(true) +}) + +it('reconciles the chat on its next send once the store can be written again', async () => { + const { host, store, onStartupReconcileFailure } = await relaunch() + lock.failing = true + await host.reconcileRestartLeases() + expect(onStartupReconcileFailure).toHaveBeenCalledOnce() + lock.failing = false + + const body = hostTestMessage('sent after a startup reconcile failed') + await expect( + host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + ).resolves.toMatchObject({ ok: true }) + + // The send is queued; delivering it starts the agent, and that start reconciles the lease. + await vi.waitFor(() => expect(hostTestState().dispatch).toHaveBeenCalledOnce(), { + timeout: 10_000 + }) + expect(store.getRecord(SESSION)?.lease.unreconciled).toBe(false) + expect(onStartupReconcileFailure).toHaveBeenCalledOnce() + // Before the relaunched directory is removed, so the child's wind-down can write its lease. + await host.flushAllStreamedEvents() +}) + +it('reports a store a newer Orca wrote without writing it, and does not reject', async () => { + const { host, store, storeDirectory, onStartupReconcileFailure } = await relaunch((json) => { + json.schemaVersion = AGENT_SESSION_STORE_SCHEMA_VERSION + 1 + }) + expect(store.readOnly).toBe(true) + const path = agentSessionStorePath(storeDirectory) + const bytes = await readFile(path) + const files = await readdir(storeDirectory) + + await expect(host.reconcileRestartLeases()).resolves.toBeUndefined() + + expect(onStartupReconcileFailure).toHaveBeenCalledOnce() + expect(onStartupReconcileFailure).toHaveBeenCalledWith( + expect.objectContaining({ message: 'agent_session_legacy_required' }) + ) + expect(store.listRecords().map((record) => record.sessionId)).toEqual([SESSION]) + expect(await readFile(path)).toEqual(bytes) + expect(await readdir(storeDirectory)).toEqual(files) +}) diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index 24735c721cc2..9cdb980684f8 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -317,6 +317,13 @@ async function installOnJournal( : {}), onEventSinkError: ({ sessionId, error }) => deps.onError?.({ scope: `structured-agent-session-journal:${sessionId}`, error }), + onStartupReconcileFailure: (error) => + deps.onError + ? deps.onError({ scope: 'structured-agent-session-startup-reconcile', error }) + : console.warn( + '[structured-agent-session] reconciling chat leases at startup failed', + error + ), ...(deps.onSessionStatusChanged ? { onSessionStatusChanged: deps.onSessionStatusChanged } : {}), ...(deps.statusSink ? { statusSink: deps.statusSink } : {}), ...(deps.hasOpenDispatch ? { hasOpenDispatch: deps.hasOpenDispatch } : {}), diff --git a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts new file mode 100644 index 000000000000..4b444eef70e1 --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts @@ -0,0 +1,110 @@ +// A profile whose chat records a newer Orca wrote opens read-only. The startup reconcile cannot +// write there, which is bookkeeping: startup must still finish rather than put the whole app into +// its degraded "Session restore failed" mode, and the file must be left exactly as it was. + +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { + AGENT_SESSION_STORE_SCHEMA_VERSION, + agentSessionStorePath +} from './agent-session-record-store-file' +import { OrcaRuntimeService } from './orca-runtime' +import { + ensureStructuredAgentSessionHost, + stopStructuredAgentSessionRuntime +} from './structured-agent-session-runtime' + +let root: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-reconcile-')) +}) + +afterEach(async () => { + await stopStructuredAgentSessionRuntime().catch(() => undefined) + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) +}) + +/** A profile whose record store a newer Orca wrote, with one chat in it. */ +async function seedNewerStore() { + const storeDirectory = join(root, 'agent-sessions') + const path = agentSessionStorePath(storeDirectory) + const record = agentSessionRecordFixture() + await mkdir(storeDirectory, { recursive: true }) + await writeFile( + path, + JSON.stringify({ + schemaVersion: AGENT_SESSION_STORE_SCHEMA_VERSION + 1, + hostId: 'local', + records: { [record.sessionId]: record }, + operations: {}, + retiredClaimKeys: [], + unusableRecords: {} + }) + ) + return { storeDirectory, path, sessionId: record.sessionId } +} + +function startupRuntime(onError?: (input: { scope: string; error: unknown }) => void) { + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these are the runtime's own protected members; the test roots the host at `root` and stubs the PTY daemon. + const internal = runtime as unknown as { + hasPersistedStructuredAgentSessionStore(): boolean + ensureStructuredAgentSessionHost(): Promise + refreshMobileSessionPtyRecords(): Promise | null> + } + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.ensureStructuredAgentSessionHost = () => + ensureStructuredAgentSessionHost({ + stateDirectory: root, + hostId: 'local', + claimKeyId: 'key-1', + resolveWorkspacePath: async () => root, + resolveEnvironment: async () => ({}), + resolveClaudeAuthPolicy: () => ({ stripAuthEnv: true }), + ...(onError ? { onError } : {}) + }) + internal.refreshMobileSessionPtyRecords = async () => new Set() + return runtime +} + +it('finishes startup over records a newer Orca wrote, reports it, and writes nothing', async () => { + const { storeDirectory, path, sessionId } = await seedNewerStore() + const bytes = await readFile(path) + const files = await readdir(storeDirectory) + const onError = vi.fn() + const runtime = startupRuntime(onError) + + // What the renderer's startup awaits through `app:prepareTerminalStartupRestoration`. + await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).resolves.toBeUndefined() + + expect(onError).toHaveBeenCalledOnce() + expect(onError).toHaveBeenCalledWith({ + scope: 'structured-agent-session-startup-reconcile', + error: expect.objectContaining({ message: 'agent_session_legacy_required' }) + }) + expect(getStructuredAgentSessionHost()?.sessionAgent(sessionId)).toBe('claude') + await stopStructuredAgentSessionRuntime() + expect(await readFile(path)).toEqual(bytes) + expect(await readdir(storeDirectory)).toEqual(files) +}) + +// The desktop installs its host with no error sink, so the failure is logged rather than dropped. +it('logs the failure when the host has no error sink', async () => { + await seedNewerStore() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + await expect( + startupRuntime().prepareStructuredAgentSessionStartupRestoration() + ).resolves.toBeUndefined() + + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] reconciling chat leases at startup failed', + expect.objectContaining({ message: 'agent_session_legacy_required' }) + ) +}) From 872f31f915ccff1766d38c30430b5038cc605714 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:12:13 -0700 Subject: [PATCH 002/107] fix(native-chat): read restored chats without waiting on lease bookkeeping With native chat on and a chat tab open at quit, the renderer's startup also awaits the chat tab restore (session.tabs.listAll). That restore re-ran the lease reconcile before reading each chat and rethrew its store failure, then recorded each restored tab as visible through a store transaction that throws on a held lock or a read-only store. Either one failed the restore, so startup still fell into "Session restore failed". Reading a chat grants no writer, so the reconcile startup and the restore run is now a reader's: createReaderReconcile never throws, answers whether every lease is settled (recovery is resolved only then; the journal opens either way), and reports each distinct failure once until a reconcile settles. Attach and agent start keep the strict reconcile. The restore's tab republish logs a failed visibility write and still publishes the tab, since a client drops every unpublished chat tab; user-driven publishes still refuse. The host dependency is renamed onLeaseReconcileFailure (scope structured-agent-session-lease-reconcile), since it now also reports for reads. --- .../structured-agent-session-host-types.ts | 6 +- .../structured-agent-session-host.ts | 10 +- ...ed-agent-session-readable-restorer.test.ts | 2 +- ...uctured-agent-session-restart-reconcile.ts | 46 +++-- ...ured-agent-session-restart-restore.test.ts | 8 +- ...tructured-agent-session-restart-restore.ts | 7 +- ...ent-session-restore-without-import.test.ts | 2 +- .../structured-agent-session-reveal.ts | 18 +- ...-session-startup-reconcile-failure.test.ts | 59 +++++-- ...tore-structured-agent-session-tabs-once.ts | 16 +- ...runtime-structured-session-restore.test.ts | 3 +- .../structured-agent-session-runtime.ts | 9 +- ...ed-agent-session-startup-reconcile.test.ts | 160 ++++++++++++++++-- 13 files changed, 270 insertions(+), 76 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index bb858f804838..e2e8e07d4fb3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -120,9 +120,9 @@ export type StructuredAgentSessionHostDeps = { /** Whether an orchestration dispatch still owns this session's worker; absent answers no. */ hasOpenDispatch?: (record: AgentSessionRecord) => boolean onEventSinkError?: (input: { sessionId: string; error: unknown }) => void - /** A startup reconcile that refused or threw. Startup carries on: every attach, send and read - * reconciles its own chat again. */ - onStartupReconcileFailure?: (failure: unknown) => void + /** A lease reconcile run for startup or a read that refused or threw, once per distinct failure. + * Startup and the read carry on: the next attach or send reconciles every lease again. */ + onLeaseReconcileFailure?: (failure: unknown) => void /** Every status projection this host publishes. `replay` marks a re-projection of state the host * already knew (restore, an arriving subscriber) rather than a fresh journal edge. */ onSessionStatusChanged?: ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 56f206db1a8d..78bc7de155ef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -10,10 +10,7 @@ import type { AgentJournalSnapshot } from '../../../shared/agent-session-journal import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' import type * as SessionWire from '../../../shared/agent-session-wire' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { - createRestartReconciler, - reconcileLeasesAtStartup -} from './structured-agent-session-restart-reconcile' +import { createRestartReconciler } from './structured-agent-session-restart-reconcile' import type { AgentSessionSubscribeInput } from './structured-agent-session-subscribers' import { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' import * as providerSupport from './structured-agent-session-provider-support' @@ -138,7 +135,7 @@ export class StructuredAgentSessionHost { flushStreamedEvents: (sessionId) => this.flushStreamedEvents(sessionId) }) this.restore = createStructuredAgentSessionHostRestore(deps, { - reconcile: this.reconcileLeases, + reconcileLeases: this.reconcileLeases, resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), hasSession: this.hasSession, @@ -231,8 +228,7 @@ export class StructuredAgentSessionHost { } } - reconcileRestartLeases = (): Promise => - reconcileLeasesAtStartup(this.reconcileLeases, this.deps.onStartupReconcileFailure) + reconcileRestartLeases = (): Promise => this.restore.reconcileRestartLeases() restoreReadableSessions = (sessionIds?: readonly string[]): Promise => this.restore.restoreReadableSessions(sessionIds) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index 5eaedfffb384..240882f17564 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -39,7 +39,7 @@ describe('StructuredAgentSessionReadableRestorer', () => { adapter: {} }, supportsRecord: () => true, - reconcile: async () => null, + reconcile: async () => true, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => false, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index b4d1380090d5..c9cba45aa59d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -45,20 +45,46 @@ export function createRestartReconciler(deps: { } } -/** Startup never fails on this verdict: a lease left unreconciled grants no writer, and the next - * attach, send or read of that chat reconciles it again. So a failure is reported, never owed. */ -export async function reconcileLeasesAtStartup( +/** The reconcile a reader runs, at startup and before each restored read: it never throws, since + * an unreconciled lease grants no writer and the next send reconciles again before it acts. + * Answers whether every lease is settled; a failure is reported once until a reconcile settles. */ +export function createReaderReconcile( reconcile: (sessionId: string) => Promise, onFailure: ((failure: unknown) => void) | undefined -): Promise { - try { - const refusal = await reconcile('startup') - if (refusal) { - onFailure?.(refusal) +): (sessionId: string) => Promise { + let reported: string | null = null + return async (sessionId) => { + let failure: unknown + try { + const refusal = await reconcile(sessionId) + if (!refusal) { + reported = null + return true + } + failure = refusal + } catch (error) { + failure = error + } + // Every read re-runs a failing reconcile, so each distinct failure is reported once. + const key = failureKey(failure) + if (key !== reported) { + reported = key + onFailure?.(failure) + } + return false + } +} + +function failureKey(failure: unknown): string { + if (typeof failure === 'object' && failure !== null) { + if ('code' in failure && failure.code) { + return String(failure.code) + } + if ('message' in failure) { + return String(failure.message) } - } catch (error) { - onFailure?.(error) } + return String(failure) } async function reconcileCurrentLeases(deps: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index c7d04ea64cc4..3bf000f9e409 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -62,7 +62,7 @@ describe('restart journal restoration', () => { const restoration = restoreStructuredAgentSessionsOnRestart({ openDeps: NO_OPEN_DEPS, records, - reconcile: async () => null, + reconcile: async () => true, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => false, @@ -103,7 +103,7 @@ describe('restart journal restoration', () => { await restoreStructuredAgentSessionsOnRestart({ openDeps: NO_OPEN_DEPS, records, - reconcile: async () => null, + reconcile: async () => true, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => false, @@ -150,7 +150,7 @@ describe('restart journal restoration', () => { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the restore reads only the record's session id here. records: [{ sessionId: 'session-1' } as AgentSessionRecord], openDeps: NO_OPEN_DEPS, - reconcile: async () => null, + reconcile: async () => true, resolveRecovery: async () => { calls.push('resolveRecovery') }, @@ -174,7 +174,7 @@ describe('restart journal restoration', () => { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the restore reads only the record's session id here. records: [{ sessionId: 'session-1' } as AgentSessionRecord], openDeps: NO_OPEN_DEPS, - reconcile: async () => null, + reconcile: async () => true, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => true, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 4e4b7f1e5ddd..02561594caa8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -13,7 +13,6 @@ import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { mapWithConcurrency } from '../../../shared/map-with-concurrency' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { @@ -28,7 +27,8 @@ export type StructuredAgentSessionReadRestoreDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { store: Pick } - reconcile: (sessionId: string) => Promise + /** Whether every lease is settled. Never throws: a read grants no writer, so bookkeeping can't block it. */ + reconcile: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean @@ -43,8 +43,7 @@ async function restoreOneStructuredAgentSessionRead( input: StructuredAgentSessionReadRestoreDeps, sessionId: string ): Promise { - const unreconciled = await input.reconcile(sessionId) - if (!unreconciled) { + if (await input.reconcile(sessionId)) { // A session latched in recovery exits here at startup, without waiting for a client. await input.resolveRecovery(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts index 226e6536c9c1..1308dbfe4a83 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts @@ -232,7 +232,7 @@ async function restore(sessionIds: readonly string[]) { await restoreStructuredAgentSessionsOnRestart({ openDeps: deps, records: sessionIds.map(recordFor), - reconcile: async () => null, + reconcile: async () => true, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: (sessionId) => sessions.has(sessionId), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 2a1899ab1ccd..96b0abd2b113 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -10,10 +10,12 @@ // a send does. And a journal it cannot open is not a refusal — the chat shows that failure with a // Retry, so the tab is worth publishing either way. +import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { adapterSupportsRecord } from './structured-agent-session-provider-support' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' +import { createReaderReconcile } from './structured-agent-session-restart-reconcile' import type { StructuredAgentSessionHostDeps, StructuredAgentSessionReveal @@ -51,23 +53,31 @@ export async function revealStructuredAgentSession( } } -/** The host's startup readable-restore sweep: reconcile, resolve, then open each chat's journal. */ +/** The host's startup readable-restore sweep: reconcile, resolve, then open each chat's journal. + * Its reconcile is a reader's, which never fails a read or startup; startup runs the same one. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], - 'openDeps' | 'supportsRecord' - > + 'openDeps' | 'supportsRecord' | 'reconcile' + > & { reconcileLeases: (sessionId: string) => Promise } ): { + reconcileRestartLeases: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise } { + const { reconcileLeases, ...rest } = wiring + const reconcile = createReaderReconcile(reconcileLeases, deps.onLeaseReconcileFailure) const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: deps, supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), - ...wiring + reconcile, + ...rest }) const gate = new StructuredAgentSessionRestartRestoreGate() return { + reconcileRestartLeases: async () => { + await reconcile('startup') + }, restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts index 0e96294872bc..5e1c8a95868c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts @@ -1,6 +1,6 @@ -// The startup reconcile is bookkeeping: a lock that gives up or a store this build may not write -// is reported, and startup carries on. Nothing is owed after it, because every chat reconciles its -// own lease again when it is sent to or read. +// The lease reconcile is bookkeeping: a lock that gives up or a store this build may not write is +// reported, and startup and every read carry on. Nothing is owed after it, because an unreconciled +// lease grants no writer and the next send reconciles every lease again before it acts. import { cp, readdir, readFile, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' @@ -73,7 +73,7 @@ async function relaunch(rewrite?: (json: Record) => void) { await writeFile(path, JSON.stringify(json)) } const store = await AgentSessionRecordStore.open({ directory: storeDirectory, hostId: 'local' }) - const onStartupReconcileFailure = vi.fn() + const onLeaseReconcileFailure = vi.fn() const host = new StructuredAgentSessionHost({ store, adapter: adapter(), @@ -82,31 +82,29 @@ async function relaunch(rewrite?: (json: Record) => void) { mintSpawnToken: () => 'spawn-next', probeOwner: async () => ({ outcome: 'pid-absent' }), now: () => NOW, - onStartupReconcileFailure + onLeaseReconcileFailure }) replaceHostTestState({ store, host }) - return { host, store, storeDirectory, onStartupReconcileFailure } + return { host, store, storeDirectory, onLeaseReconcileFailure } } it('reports a startup reconcile whose store write fails, and does not reject', async () => { - const { host, store, onStartupReconcileFailure } = await relaunch() + const { host, store, onLeaseReconcileFailure } = await relaunch() lock.failing = true await expect(host.reconcileRestartLeases()).resolves.toBeUndefined() - expect(onStartupReconcileFailure).toHaveBeenCalledOnce() - expect(onStartupReconcileFailure).toHaveBeenCalledWith( - expect.objectContaining({ code: 'ELOCKED' }) - ) + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() + expect(onLeaseReconcileFailure).toHaveBeenCalledWith(expect.objectContaining({ code: 'ELOCKED' })) // Nothing was adjudicated, so the lease still grants no writer. expect(store.getRecord(SESSION)?.lease.unreconciled).toBe(true) }) it('reconciles the chat on its next send once the store can be written again', async () => { - const { host, store, onStartupReconcileFailure } = await relaunch() + const { host, store, onLeaseReconcileFailure } = await relaunch() lock.failing = true await host.reconcileRestartLeases() - expect(onStartupReconcileFailure).toHaveBeenCalledOnce() + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() lock.failing = false const body = hostTestMessage('sent after a startup reconcile failed') @@ -119,13 +117,13 @@ it('reconciles the chat on its next send once the store can be written again', a timeout: 10_000 }) expect(store.getRecord(SESSION)?.lease.unreconciled).toBe(false) - expect(onStartupReconcileFailure).toHaveBeenCalledOnce() + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() // Before the relaunched directory is removed, so the child's wind-down can write its lease. await host.flushAllStreamedEvents() }) it('reports a store a newer Orca wrote without writing it, and does not reject', async () => { - const { host, store, storeDirectory, onStartupReconcileFailure } = await relaunch((json) => { + const { host, store, storeDirectory, onLeaseReconcileFailure } = await relaunch((json) => { json.schemaVersion = AGENT_SESSION_STORE_SCHEMA_VERSION + 1 }) expect(store.readOnly).toBe(true) @@ -135,11 +133,38 @@ it('reports a store a newer Orca wrote without writing it, and does not reject', await expect(host.reconcileRestartLeases()).resolves.toBeUndefined() - expect(onStartupReconcileFailure).toHaveBeenCalledOnce() - expect(onStartupReconcileFailure).toHaveBeenCalledWith( + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() + expect(onLeaseReconcileFailure).toHaveBeenCalledWith( expect.objectContaining({ message: 'agent_session_legacy_required' }) ) expect(store.listRecords().map((record) => record.sessionId)).toEqual([SESSION]) expect(await readFile(path)).toEqual(bytes) expect(await readdir(storeDirectory)).toEqual(files) }) + +it('restores a chat for reading while the reconcile keeps failing, and reports it once', async () => { + const { host, store, onLeaseReconcileFailure } = await relaunch() + lock.failing = true + await host.reconcileRestartLeases() + + await expect(host.restoreReadableSessions([SESSION])).resolves.toBeUndefined() + + expect(host.hasSession(SESSION)).toBe(true) + expect(store.getRecord(SESSION)?.lease.unreconciled).toBe(true) + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() + expect(onLeaseReconcileFailure).toHaveBeenCalledWith(expect.objectContaining({ code: 'ELOCKED' })) +}) + +it('restores a chat for reading from a store a newer Orca wrote', async () => { + const { host, storeDirectory, onLeaseReconcileFailure } = await relaunch((json) => { + json.schemaVersion = AGENT_SESSION_STORE_SCHEMA_VERSION + 1 + }) + const path = agentSessionStorePath(storeDirectory) + const bytes = await readFile(path) + + await expect(host.restoreReadableSessions([SESSION])).resolves.toBeUndefined() + + expect(host.hasSession(SESSION)).toBe(true) + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() + expect(await readFile(path)).toEqual(bytes) +}) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 8d29a0d360b9..476b07820552 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -83,7 +83,8 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu agent: session.agent, sessionId, activate: false, - notify: false + notify: false, + visibilityWriteMayFail: true }) } const wasUnverifiable = this.structuredAgentSessionInventoryUnverifiable @@ -105,14 +106,25 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu replacesSessionId?: string /** The host tab id a create reserved; a session that already has a tab keeps its own. */ tabId?: string + /** A restore's republish: a failed visibility write is logged and the tab still publishes, + * since a client drops every unpublished chat tab. */ + visibilityWriteMayFail?: boolean }): Promise { const host = getStructuredAgentSessionHost() if (typeof host?.setSessionTabVisibility === 'function') { - await host.setSessionTabVisibility( + const written = host.setSessionTabVisibility( input.sessionId, true, ...(input.tabId ? [input.tabId] : []) ) + await (input.visibilityWriteMayFail + ? written.catch((error) => + console.warn('[structured-agent-session] recording a restored chat tab failed', { + sessionId: input.sessionId, + error + }) + ) + : written) } const existing = this.mobileSessionTabsByWorktree.get(input.workspaceId) const id = `agent-session:${input.sessionId}` diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 1d8105d07f81..80b8e4f3be65 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -356,7 +356,8 @@ describe('structured session cold restoration', () => { sessionId: 'restored-claude', agent: 'claude', activate: false, - notify: false + notify: false, + visibilityWriteMayFail: true }) const restored = await runtime.listMobileSessionTabs('id:workspace-1') diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index 9cdb980684f8..0c6b392970d4 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -317,13 +317,10 @@ async function installOnJournal( : {}), onEventSinkError: ({ sessionId, error }) => deps.onError?.({ scope: `structured-agent-session-journal:${sessionId}`, error }), - onStartupReconcileFailure: (error) => + onLeaseReconcileFailure: (error) => deps.onError - ? deps.onError({ scope: 'structured-agent-session-startup-reconcile', error }) - : console.warn( - '[structured-agent-session] reconciling chat leases at startup failed', - error - ), + ? deps.onError({ scope: 'structured-agent-session-lease-reconcile', error }) + : console.warn('[structured-agent-session] reconciling chat leases failed', error), ...(deps.onSessionStatusChanged ? { onSessionStatusChanged: deps.onSessionStatusChanged } : {}), ...(deps.statusSink ? { statusSink: deps.statusSink } : {}), ...(deps.hasOpenDispatch ? { hasOpenDispatch: deps.hasOpenDispatch } : {}), diff --git a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts index 4b444eef70e1..8fecec36a2ce 100644 --- a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts +++ b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts @@ -5,8 +5,15 @@ import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecord } from '../../shared/agent-session-record' +import type { RuntimeMobileSessionTabsSnapshot } from '../../shared/runtime-types' import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' +import type * as FileTransactionLock from '../file-transaction-lock' +import { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' +import { openAgentSessionJournal } from '../native-chat/agent-session-journal/journal-store-factory' +import { journalIdentityFor } from '../native-chat/agent-session-wire/structured-agent-session-attach' +import { attachParamsForRecord } from '../native-chat/agent-session-wire/structured-agent-session-conversation-open' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { AGENT_SESSION_STORE_SCHEMA_VERSION, @@ -18,6 +25,22 @@ import { stopStructuredAgentSessionRuntime } from './structured-agent-session-runtime' +const lock = vi.hoisted(() => ({ failing: false })) + +vi.mock('../file-transaction-lock', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + withFileTransactionLock: (...args: Parameters) => + lock.failing + ? // What proper-lockfile throws once its retries give up. + Promise.reject( + Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED' }) + ) + : actual.withFileTransactionLock(...args) + } +}) + let root: string beforeEach(async () => { @@ -25,13 +48,16 @@ beforeEach(async () => { }) afterEach(async () => { + lock.failing = false await stopStructuredAgentSessionRuntime().catch(() => undefined) vi.restoreAllMocks() await rm(root, { recursive: true, force: true }) }) -/** A profile whose record store a newer Orca wrote, with one chat in it. */ -async function seedNewerStore() { +const PROMPT = 'add a retry' + +/** A profile's record store with one chat in it; one a newer Orca wrote opens read-only here. */ +async function seedStore(options: { newer: boolean; visible?: boolean }) { const storeDirectory = join(root, 'agent-sessions') const path = agentSessionStorePath(storeDirectory) const record = agentSessionRecordFixture() @@ -39,28 +65,61 @@ async function seedNewerStore() { await writeFile( path, JSON.stringify({ - schemaVersion: AGENT_SESSION_STORE_SCHEMA_VERSION + 1, + schemaVersion: AGENT_SESSION_STORE_SCHEMA_VERSION + (options.newer ? 1 : 0), hostId: 'local', records: { [record.sessionId]: record }, operations: {}, retiredClaimKeys: [], - unusableRecords: {} + unusableRecords: {}, + ...(options.visible ? { visibleSessionIds: [record.sessionId] } : {}) }) ) - return { storeDirectory, path, sessionId: record.sessionId } + return { storeDirectory, path, record, sessionId: record.sessionId } } -function startupRuntime(onError?: (input: { scope: string; error: unknown }) => void) { +/** The chat's history as the last run left it: one prompt, accepted. */ +async function seedHistory(record: AgentSessionRecord): Promise { + const fence = record.lease.runtimeFence + const database = JournalHostDatabase.open(root) + const journal = await openAgentSessionJournal({ + identity: journalIdentityFor( + record, + attachParamsForRecord(record, { clientOperationId: 'seed', expectedRuntimeFence: fence }) + ), + database + }) + await journal.appendSubmission({ + clientMessageId: 'client-1', + payloadFingerprint: 'fp-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: PROMPT }] }, + fence, + handoverRecorded: true + }) + await journal.resolveDispatch({ + clientMessageId: 'client-1', + fence, + state: 'accepted', + providerIdentity: null + }) + await journal.close() + database.close() +} + +function startupRuntime( + onError?: (input: { scope: string; error: unknown }) => void, + afterInstall?: () => void +) { const runtime = new OrcaRuntimeService() // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these are the runtime's own protected members; the test roots the host at `root` and stubs the PTY daemon. const internal = runtime as unknown as { hasPersistedStructuredAgentSessionStore(): boolean ensureStructuredAgentSessionHost(): Promise refreshMobileSessionPtyRecords(): Promise | null> + mobileSessionTabsByWorktree: Map } internal.hasPersistedStructuredAgentSessionStore = () => true - internal.ensureStructuredAgentSessionHost = () => - ensureStructuredAgentSessionHost({ + internal.ensureStructuredAgentSessionHost = async () => { + const installed = await ensureStructuredAgentSessionHost({ stateDirectory: root, hostId: 'local', claimKeyId: 'key-1', @@ -69,23 +128,26 @@ function startupRuntime(onError?: (input: { scope: string; error: unknown }) => resolveClaudeAuthPolicy: () => ({ stripAuthEnv: true }), ...(onError ? { onError } : {}) }) + afterInstall?.() + return installed + } internal.refreshMobileSessionPtyRecords = async () => new Set() - return runtime + return { runtime, publishedTabs: () => internal.mobileSessionTabsByWorktree.get('workspace-1') } } it('finishes startup over records a newer Orca wrote, reports it, and writes nothing', async () => { - const { storeDirectory, path, sessionId } = await seedNewerStore() + const { storeDirectory, path, sessionId } = await seedStore({ newer: true }) const bytes = await readFile(path) const files = await readdir(storeDirectory) const onError = vi.fn() - const runtime = startupRuntime(onError) + const { runtime } = startupRuntime(onError) // What the renderer's startup awaits through `app:prepareTerminalStartupRestoration`. await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).resolves.toBeUndefined() expect(onError).toHaveBeenCalledOnce() expect(onError).toHaveBeenCalledWith({ - scope: 'structured-agent-session-startup-reconcile', + scope: 'structured-agent-session-lease-reconcile', error: expect.objectContaining({ message: 'agent_session_legacy_required' }) }) expect(getStructuredAgentSessionHost()?.sessionAgent(sessionId)).toBe('claude') @@ -96,15 +158,81 @@ it('finishes startup over records a newer Orca wrote, reports it, and writes not // The desktop installs its host with no error sink, so the failure is logged rather than dropped. it('logs the failure when the host has no error sink', async () => { - await seedNewerStore() + await seedStore({ newer: true }) const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) await expect( - startupRuntime().prepareStructuredAgentSessionStartupRestoration() + startupRuntime().runtime.prepareStructuredAgentSessionStartupRestoration() ).resolves.toBeUndefined() expect(warn).toHaveBeenCalledWith( - '[structured-agent-session] reconciling chat leases at startup failed', + '[structured-agent-session] reconciling chat leases failed', expect.objectContaining({ message: 'agent_session_legacy_required' }) ) }) + +// With native chat on, the renderer's startup also awaits the chat tab restore (`session.tabs.listAll`), +// which reads every chat whose tab was open at quit. That read must not wait on the lease bookkeeping. +describe('restoring the chat tabs open at quit', () => { + async function expectChatRestored( + runtime: OrcaRuntimeService, + publishedTabs: () => RuntimeMobileSessionTabsSnapshot | undefined, + sessionId: string + ) { + await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() + expect(publishedTabs()?.tabs.map((tab) => tab.id)).toEqual([`agent-session:${sessionId}`]) + const host = getStructuredAgentSessionHost() + expect(JSON.stringify((await host!.journalSnapshot(sessionId)).items)).toContain(PROMPT) + } + + /** The reconcile failure once, though startup and the read both ran it; the tab write per chat. */ + function expectReported( + onError: ReturnType, + warn: ReturnType, + sessionId: string, + failure: Record + ) { + expect(onError).toHaveBeenCalledOnce() + expect(onError).toHaveBeenCalledWith({ + scope: 'structured-agent-session-lease-reconcile', + error: expect.objectContaining(failure) + }) + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] recording a restored chat tab failed', + { sessionId, error: expect.objectContaining(failure) } + ) + } + + it('lists and reads a chat from records a newer Orca wrote, and writes nothing', async () => { + const { storeDirectory, path, record } = await seedStore({ newer: true, visible: true }) + await seedHistory(record) + const bytes = await readFile(path) + const files = await readdir(storeDirectory) + const onError = vi.fn() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { runtime, publishedTabs } = startupRuntime(onError) + + await expectChatRestored(runtime, publishedTabs, record.sessionId) + + expectReported(onError, warn, record.sessionId, { message: 'agent_session_legacy_required' }) + await stopStructuredAgentSessionRuntime() + expect(await readFile(path)).toEqual(bytes) + expect(await readdir(storeDirectory)).toEqual(files) + }) + + it('lists and reads a chat while the record store lock keeps failing', async () => { + const { record } = await seedStore({ newer: false, visible: true }) + await seedHistory(record) + const onError = vi.fn() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + // Free when the store opens, then held for good: every later write gives up. + const { runtime, publishedTabs } = startupRuntime(onError, () => { + lock.failing = true + }) + + await expectChatRestored(runtime, publishedTabs, record.sessionId) + + expectReported(onError, warn, record.sessionId, { code: 'ELOCKED' }) + lock.failing = false + }) +}) From e5d1ab2563f1c5b3e01d889d87af0f3546edc520 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:48:43 -0700 Subject: [PATCH 003/107] fix(native-chat): list chat tabs without opening every chat, and stop startup waiting on the lease check At startup the chat tab list, and the window's first paint behind it, waited for two things: the startup lease check over every saved chat, and opening and replaying the full history of every listed chat. The list itself was then taken from the chats that had opened, so a chat whose history was missing or could not be opened lost its tab after a restart, and one failed open or lease check failed the whole list. Now the tab list comes from the durable records and the tab table, in tab order, filtered to chats this host serves. It opens nothing. Each chat's history is opened after the list is published, by the same restore pass as before, not awaited; a failure there is logged and leaves the list alone. The startup lease check is started and never awaited: neither the list nor paint needs leases. Two things did rely on the startup wait: - a write to a chat whose lease is still the previous run's. Every write (send, Stop, prompt answer, option, goal, rewind, clear, compact, queued drafts, restart-offer resume) enters its chat's queue through the lease check now: it waits for that chat's check, or runs it, when the lease is unreconciled, and enqueues at once otherwise. Attach and agent start already ran the check themselves. - the structured worker release, which the renderer runs right after the startup step. It now runs the check itself before it observes the worker. No schema change, no saved status. Status rows now arrive per chat just after paint instead of all at paint. --- .../structured-agent-session-host-tabs.ts | 33 ++- .../structured-agent-session-host.test.ts | 2 +- .../structured-agent-session-host.ts | 9 +- ...gent-session-persisted-tab-listing.test.ts | 122 +++++++++ ...red-agent-session-provider-restore.test.ts | 2 +- ...ctured-agent-session-refusal-retry.test.ts | 5 + ...uctured-agent-session-restart-reconcile.ts | 19 ++ ...ent-session-restored-status-parity.test.ts | 89 +++++++ ...-agent-session-startup-listing-test-rig.ts | 242 +++++++++++++++++ ...red-agent-session-surface-lifetime.test.ts | 2 +- ...red-agent-session-write-lease-wait.test.ts | 109 ++++++++ ...structured-agent-session-create-support.ts | 18 +- ...tore-structured-agent-session-tabs-once.ts | 18 +- ...runtime-structured-session-restore.test.ts | 7 +- .../worker/worker-release-completion.ts | 6 +- ...orker-release-structured-reconcile.test.ts | 62 +++++ .../structured-worker-tab-retirement.test.ts | 2 + ...ed-agent-session-startup-reconcile.test.ts | 11 +- ...-agent-session-tab-listing-startup.test.ts | 245 ++++++++++++++++++ 19 files changed, 968 insertions(+), 35 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts create mode 100644 src/main/runtime/structured-agent-session-tab-listing-startup.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts index 8a18040daf35..ee13814b485b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts @@ -1,4 +1,5 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { adapterSupportsRecord } from './structured-agent-session-provider-support' /** * Chat-tab visibility is the deletion funnel: every path that removes a chat as a user-facing @@ -37,15 +38,27 @@ export type StructuredAgentSessionTab = { agent: AgentSessionRecord['provider'] } -export function listStructuredAgentSessionTabs( - sessions: ReadonlyMap< - string, - { params: { location: { workspaceId: string }; provider: AgentSessionRecord['provider'] } } - > +/** Tabs from durable state alone: each requested id, once, in the order given, that has a record + * this host serves. Opens nothing: a chat whose history is missing or unreadable keeps its tab and + * explains itself when read. */ +export function listPersistedSessionTabs( + deps: { + store: { getRecord: (sessionId: string) => AgentSessionRecord | null } + adapter: Parameters[0] + }, + sessionIds: readonly string[] ): StructuredAgentSessionTab[] { - return [...sessions.entries()].map(([sessionId, session]) => ({ - sessionId, - workspaceId: session.params.location.workspaceId, - agent: session.params.provider - })) + const tabs = new Map() + for (const sessionId of sessionIds) { + const record = deps.store.getRecord(sessionId) + if (!record || tabs.has(sessionId) || !adapterSupportsRecord(deps.adapter, record)) { + continue + } + tabs.set(sessionId, { + sessionId, + workspaceId: record.location.workspaceId, + agent: record.provider + }) + } + return [...tabs.values()] } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts index 590f69c79840..3bb69fc16b7f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts @@ -610,7 +610,7 @@ describe('restart', () => { const restoreReads = listRecords.mock.calls.length await host.restoreReadableSessions() - expect(host.listSessionTabs()).toEqual([ + expect(host.listSessionTabs([SESSION])).toEqual([ { sessionId: SESSION, workspaceId: 'workspace-1', agent: 'codex' } ]) const history = await host.history({ sessionId: SESSION, direction: 'tail' }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 56f206db1a8d..f0a3ad6ca450 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -12,7 +12,8 @@ import type * as SessionWire from '../../../shared/agent-session-wire' import type { AgentSessionAttachParams } from './structured-agent-session-attach' import { createRestartReconciler, - reconcileLeasesAtStartup + reconcileLeasesAtStartup, + serializeAfterLeaseCheck } from './structured-agent-session-restart-reconcile' import type { AgentSessionSubscribeInput } from './structured-agent-session-subscribers' import { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' @@ -215,7 +216,8 @@ export class StructuredAgentSessionHost { supportsCreate = (location: AgentSessionExecutionLocation, agent: string): boolean => providerSupport.adapterSupportsCreate(this.deps.adapter, location, agent) - listSessionTabs = () => sessionTabs.listStructuredAgentSessionTabs(this.sessions) + /** From the record store and the given tab ids; opens no conversation. */ + listSessionTabs = (ids: readonly string[]) => sessionTabs.listPersistedSessionTabs(this.deps, ids) getPersistedVisibleSessionTabIndex = () => this.deps.store.getVisibleSessionTabIndex() getSessionTabId = (sessionId: string): string | null => this.deps.store.getSessionTabId(sessionId) @@ -274,7 +276,8 @@ export class StructuredAgentSessionHost { publish: (sessionId, journal) => this.subscribers.publish(sessionId, journal), flushStreamedEvents: this.flushStreamedEvents, conversation: this.lifetime.conversation, - serialize: (sessionId, task) => this.serialize(sessionId, task), + // Every write funnels through here, so none acts on a lease the startup check has not reached. + serialize: serializeAfterLeaseCheck(this.deps.store, this.reconcileLeases, this.serialize), openConversation: this.conversationDelivery.open, ensureAgent: (sessionId) => ensureStructuredAgentSessionAgentForOperation(this.attachContext(), sessionId), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts new file mode 100644 index 000000000000..f14a53ee673a --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts @@ -0,0 +1,122 @@ +// A restarted host lists a chat's tab from its record and the tab table, never from whether its +// history opened: a chat with no history on disk, and one whose history cannot be read, both keep +// their tab. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { isAgentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { + createStartupRig, + type StartupRig +} from './structured-agent-session-startup-listing-test-rig' + +let rig: StartupRig + +beforeEach(async () => { + rig = await createStartupRig() +}) + +afterEach(async () => { + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +const tab = (sessionId: string, workspaceId = 'workspace-1') => ({ + sessionId, + workspaceId, + agent: 'codex' +}) + +describe('the tab list after a restart', () => { + it('lists every chat it is given without opening one', async () => { + await rig.chat('session-a', { message: 'one' }) + await rig.chat('session-b', { message: 'two' }) + await rig.crash() + const host = await rig.boot() + + expect(host.listSessionTabs(['session-a', 'session-b'])).toEqual([ + tab('session-a'), + tab('session-b') + ]) + expect(rig.historyFilePath).not.toHaveBeenCalled() + }) + + it('keeps a chat with no history on disk, which then reads empty and takes a send (T3)', async () => { + // Its first start failed before the conversation opened, so no history was ever written. + rig.acquire.mockRejectedValueOnce(new Error('spawn failed')) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + await expect(rig.chat('session-new')).rejects.toThrow(/attach refused/) + await rig.store.setSessionTabVisibility('session-new', true) + await rig.chat('session-old', { message: 'hello' }) + await rig.crash() + const host = await rig.boot() + const ids = ['session-new', 'session-old'] + + await host.restoreReadableSessions(ids) + + expect(host.listSessionTabs(ids)).toEqual([tab('session-new'), tab('session-old')]) + // The restore founds no history for it; the tab stands anyway. + expect(rig.opensOf('session-new')).toBe(0) + const page = await host.history({ sessionId: 'session-new', direction: 'tail' }) + expect(page.ok && page.page.items).toEqual([]) + const sent = await rig.send('session-new', 'first words') + expect(sent).toMatchObject({ ok: true }) + await vi.waitFor(() => expect(rig.dispatch).toHaveBeenCalledOnce()) + const after = await host.history({ sessionId: 'session-new', direction: 'tail' }) + expect(JSON.stringify(after.ok && after.page)).toContain('first words') + }) + + it('keeps a chat whose history cannot be opened, and its read says why (T4)', async () => { + await rig.chat('session-bad', { message: 'lost' }) + await rig.chat('session-good', { message: 'kept' }) + await rig.crash() + const host = await rig.boot() + rig.historyFilePath.mockImplementation(async (sessionId) => { + if (sessionId === 'session-bad') { + throw new Error('EACCES: permission denied') + } + return null + }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const ids = ['session-bad', 'session-good'] + + await host.restoreReadableSessions(ids).catch(() => undefined) + + expect(host.listSessionTabs(ids)).toEqual([tab('session-bad'), tab('session-good')]) + const read = await host.history({ sessionId: 'session-bad', direction: 'tail' }).then( + () => null, + (error: unknown) => error + ) + expect(isAgentSessionRefusalError(read) && read.refusal.code).toBe( + 'agent_session_journal_unreadable' + ) + }) + + it('lists in the order it is given, once each, whatever order the records are in (T7)', async () => { + await rig.chat('session-c') + await rig.chat('session-a') + await rig.chat('session-b') + await rig.crash() + const host = await rig.boot() + + expect(host.listSessionTabs(['session-b', 'session-c', 'session-a', 'session-b'])).toEqual([ + tab('session-b'), + tab('session-c'), + tab('session-a') + ]) + }) + + it('lists no chat without a record, and none this host cannot serve (T10, regression guard)', async () => { + await rig.chat('session-served', { message: 'here' }) + await rig.chat('session-gated', { workspaceId: 'workspace-gated', message: 'there' }) + await rig.crash() + const host = await rig.boot() + rig.unsupportedWorkspaceIds.add('workspace-gated') + const ids = ['session-served', 'session-gated', 'session-missing'] + + await host.restoreReadableSessions(ids) + + expect(host.listSessionTabs(ids)).toEqual([tab('session-served')]) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts index e5025551360b..e21663f1377e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts @@ -102,7 +102,7 @@ describe('structured session provider restore', () => { await restarted.restoreReadableSessions() - expect(restarted.listSessionTabs()).toEqual([ + expect(restarted.listSessionTabs([HOST_TEST_SESSION])).toEqual([ { sessionId: HOST_TEST_SESSION, workspaceId: 'workspace-1', agent: 'claude' } ]) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts index 4c2db496c3cf..7e0f530d179a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts @@ -335,9 +335,14 @@ describe('agentSessionRefusalOperationState host oracle', () => { ...current, lease: { ...current.lease, unreconciled: true } })) + // A write runs the lease check first; only one that cannot settle leaves this refusal. + const stuck = vi + .spyOn(reconciling.store, 'reconcileOnRestart') + .mockRejectedValue(new Error('execution_owner_reconciling')) const spec = { method, operationId: operationId() } record( await assertHostAgreement(reconciling, spec, 'execution_owner_reconciling', async () => { + stuck.mockRestore() await setLease(reconciling, (current) => ({ ...current, lease: { ...current.lease, unreconciled: false } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index b4d1380090d5..26207a29e9bb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -61,6 +61,25 @@ export async function reconcileLeasesAtStartup( } } +/** The chat's queue, entered by a write only once that chat's lease is checked. Startup does not + * wait for the lease check, so a write can reach a chat whose lease is still the previous run's; + * it waits for that check, or runs it. With none owed it enqueues at once, keeping its place. A + * check that fails gates nothing: the lease stays unreconciled, which admits no writer, and the + * next start checks again. */ +export function serializeAfterLeaseCheck( + store: Pick, + reconcile: (sessionId: string) => Promise, + serialize: (sessionId: string, task: () => Promise) => Promise +): (sessionId: string, task: () => Promise) => Promise { + return (sessionId, task) => + store.getRecord(sessionId)?.lease.unreconciled + ? reconcile(sessionId).then( + () => serialize(sessionId, task), + () => serialize(sessionId, task) + ) + : serialize(sessionId, task) +} + async function reconcileCurrentLeases(deps: { store: AgentSessionRecordStore probe: (record: AgentSessionRecord) => Promise diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts new file mode 100644 index 000000000000..b0c033c2f84b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts @@ -0,0 +1,89 @@ +// Status rows now arrive after the tab list, from the post-listing restore pass. Each must end +// exactly where a fresh host lands when the same chat is opened directly by a read (T8, regression +// guard: the pass is main's own open path; this pins that moving it after the list changed nothing). + +import { cp } from 'node:fs/promises' +import { afterEach, expect, it, vi } from 'vitest' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { + createStartupRig, + latestStatus, + type StartupRig +} from './structured-agent-session-startup-listing-test-rig' + +const rigs: StartupRig[] = [] + +afterEach(async () => { + for (const rig of rigs.splice(0)) { + await rig.dispose() + } + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +const CORPUS = ['session-settled', 'session-quiet', 'session-crashed', 'session-pending'] + +/** The chats a restart finds: two a clean quit settled, one cut off mid-turn by a crash, and one + * whose send the provider only admitted, left pending below a fence that has since moved. */ +async function seedCorpus(rig: StartupRig): Promise { + await rig.chat('session-settled', { message: 'finished work' }) + await rig.chat('session-quiet') + await rig.host.flushAllStreamedEvents() + await rig.boot() + await rig.chat('session-crashed', { message: 'cut off' }) + rig.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await rig.chat('session-pending', { message: 'only admitted' }) + await rig.crash() + await rig.store.transitionHandoff('session-pending', (record) => ({ + ...record, + lease: { ...record.lease, runtimeFence: record.lease.runtimeFence + 1 } + })) +} + +it('ends every chat where a direct read of it lands, and shows no pre-crash work (T8)', async () => { + const passRig = await createStartupRig() + rigs.push(passRig) + await seedCorpus(passRig) + // One copy of the files for each boot, taken while nothing is writing. + closeTestJournalHostDatabases() + const readRoot = `${passRig.root}-read` + await cp(passRig.root, readRoot, { recursive: true }) + const readRig = await createStartupRig(readRoot) + rigs.push(readRig) + + // Startup's shape: the lease check started, the list answered, then the pass. + const passHost = await passRig.boot() + const passCheck = passHost.reconcileRestartLeases() + await passHost.restoreReadableSessions(CORPUS) + await passCheck + + // The same run, where the window reads each chat instead. + const readHost = readRig.host + const readCheck = readHost.reconcileRestartLeases() + for (const sessionId of CORPUS) { + await readHost.history({ sessionId, direction: 'tail' }) + } + await readCheck + + // `updatedAt` is when the row was projected, a wall-clock read that differs between two boots. + const rows = (rig: StartupRig): Record> => + Object.fromEntries( + CORPUS.flatMap((sessionId) => { + const summary = latestStatus(rig, sessionId) + if (!summary) { + return [] + } + const { updatedAt: _updatedAt, ...row } = summary + return [[sessionId, row]] + }) + ) + await vi.waitFor(() => expect(rows(passRig)).toEqual(rows(readRig))) + for (const sessionId of CORPUS) { + expect(latestStatus(passRig, sessionId)).toBeDefined() + } + const crashedRows = passRig.statusEvents.flatMap((event) => + event.type === 'status' && event.session.sessionId === 'session-crashed' ? [event.session] : [] + ) + expect(crashedRows.map((row) => row.status)).not.toContain('working') +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts new file mode 100644 index 000000000000..479f9b4df51b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts @@ -0,0 +1,242 @@ +// Real hosts over one real record store and journal database, booted in turn the way app runs +// follow each other, for the tests of what a restarted host lists and owes before anyone opens a +// chat. Every journal open calls the adapter's `historyFilePath` once, so that is the open counter, +// and holding it holds the open. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { + AgentSessionMutationEnvelope, + AgentSessionStatusEvent +} from '../../../shared/agent-session-wire' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import type { AgentSessionAttachParams } from './structured-agent-session-attach' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { + HOST_TEST_LOCATION, + HOST_TEST_NOW, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +export const STARTUP_CALLER = { callerKey: 'client-1' } + +export type StartupRig = { + root: string + store: AgentSessionRecordStore + host: StructuredAgentSessionHost + acquire: Mock + dispatch: Mock + /** Called once per journal open, with the session id; replace its implementation to hold one. */ + historyFilePath: Mock<(sessionId: string) => Promise> + probeOwner: Mock<(record: AgentSessionRecord) => Promise> + /** Workspaces this host's adapter does not serve, as a platform gate would. */ + unsupportedWorkspaceIds: Set + statusEvents: AgentSessionStatusEvent[] + /** Creates the chat, lists its tab unless told not to, and sends `message` when one is given, + * waiting for its dispatch. */ + chat: ( + sessionId: string, + options?: { workspaceId?: string; listed?: boolean; message?: string } + ) => Promise + send: (sessionId: string, text: string) => ReturnType + opensOf: (sessionId: string) => number + /** A crash: nothing settles or flushes; the process's timers and handles simply stop. */ + crash: () => Promise + /** A new app run over the same files; the counters and status stream start empty. */ + boot: (deps?: Partial) => Promise + dispose: () => Promise +} + +export function startupSendEnvelope( + sessionId: string, + body: ReturnType +): AgentSessionMutationEnvelope { + return { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId, + fields: { body } + }) + } +} + +function startupAttachParams( + sessionId: string, + workspaceId: string, + fence: number | null +): AgentSessionAttachParams { + return hostTestAttachParams(fence, { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: '' + }, + location: { ...HOST_TEST_LOCATION, workspaceId }, + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + }) +} + +export async function createStartupRig( + root?: string, + deps: Partial = {} +): Promise { + resetHostTestOperationIds() + const stateRoot = root ?? (await mkdtemp(join(tmpdir(), 'orca-startup-listing-'))) + const directory = join(stateRoot, 'store') + const statusEvents: AgentSessionStatusEvent[] = [] + const historyFilePath = vi.fn(async (_sessionId: string): Promise => null) + const probeOwner: StartupRig['probeOwner'] = vi.fn(async () => ({ outcome: 'pid-absent' })) + const unsupportedWorkspaceIds = new Set() + let ordinal = 0 + const acquire: StartupRig['acquire'] = vi.fn(async ({ fence, spawnToken, identity }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${identity.sessionId}-${fence}`, + handle: { provider: 'codex' as const, threadId: `thread-${identity.sessionId}` }, + origin: rig.store.getRecord(identity.sessionId)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: HOST_TEST_NOW + } + })) + const dispatch: StartupRig['dispatch'] = vi.fn(async (input) => { + ordinal += 1 + return { + state: 'accepted', + providerIdentity: { + provider: 'codex', + threadId: `thread-${input.sessionId}`, + turnId: `turn-${ordinal}`, + ordinal + } + } + }) + const open = async (overrides: Partial) => { + const store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) + const host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: async () => ({ cancelled: true }), + answerPrompt: async ({ commit }) => commit(), + setOption: async () => undefined, + historyFilePath: ({ identity }) => historyFilePath(identity.sessionId), + supportsCreate: (location, agent) => + agent === 'codex' && !unsupportedWorkspaceIds.has(location.workspaceId) + }, + journalDatabase: openTestJournalHostDatabase(stateRoot), + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + probeOwner, + now: () => HOST_TEST_NOW, + ...deps, + ...overrides + }) + host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) + return { store, host } + } + const first = await open({}) + const rig: StartupRig = { + root: stateRoot, + ...first, + acquire, + dispatch, + historyFilePath, + probeOwner, + unsupportedWorkspaceIds, + statusEvents, + chat: async (sessionId, options = {}) => { + const attached = await rig.host.attach( + STARTUP_CALLER, + startupAttachParams( + sessionId, + options.workspaceId ?? 'workspace-1', + rig.store.getRecord(sessionId)?.lease.runtimeFence ?? null + ) + ) + if (!attached.ok) { + throw new Error(`attach refused: ${attached.refusal.code}`) + } + if (options.listed !== false) { + await rig.store.setSessionTabVisibility(sessionId, true) + } + if (options.message === undefined) { + return + } + const dispatched = dispatch.mock.calls.length + const sent = await rig.send(sessionId, options.message) + if (!sent.ok) { + throw new Error(`send refused: ${sent.refusal.code}`) + } + await vi.waitFor(() => expect(dispatch.mock.calls.length).toBeGreaterThan(dispatched)) + }, + send: (sessionId, text) => { + const body = hostTestMessage(text) + return rig.host.send(STARTUP_CALLER, { body, envelope: startupSendEnvelope(sessionId, body) }) + }, + opensOf: (sessionId) => historyFilePath.mock.calls.filter(([id]) => id === sessionId).length, + crash: async () => { + const { runtimeState, lifetime, sessions } = rig.host.collaboratorsForTests() + await runtimeState.stopLeaseRenewal() + lifetime.dispose() + for (const session of sessions.values()) { + await session.journal.close() + } + sessions.clear() + }, + boot: async (overrides = {}) => { + // What the new run saw, and nothing the previous one did. + historyFilePath.mockClear() + acquire.mockClear() + dispatch.mockClear() + probeOwner.mockClear() + statusEvents.length = 0 + const next = await open(overrides) + rig.store = next.store + rig.host = next.host + return next.host + }, + dispose: async () => { + await rig.host.flushAllStreamedEvents().catch(() => undefined) + if (!root) { + await rm(stateRoot, { recursive: true, force: true }) + } + } + } + return rig +} + +/** The newest row the status stream carried for a session. */ +export function latestStatus(rig: StartupRig, sessionId: string) { + for (const event of rig.statusEvents.toReversed()) { + if (event.type === 'status' && event.session.sessionId === sessionId) { + return event.session + } + if (event.type === 'snapshot') { + const found = event.sessions.find((session) => session.sessionId === sessionId) + if (found) { + return found + } + } + } + return undefined +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index 586f8c826300..1cc82cc7b28a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -428,7 +428,7 @@ describe('startup', () => { // The record is readable — the tab comes back, history answers — and nothing is running. expect(acquire).not.toHaveBeenCalled() - expect(host.listSessionTabs()).toEqual([ + expect(host.listSessionTabs([SESSION])).toEqual([ { sessionId: SESSION, workspaceId: 'workspace-1', agent: 'codex' } ]) expect((await host.history({ sessionId: SESSION, direction: 'tail' })).ok).toBe(true) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts new file mode 100644 index 000000000000..93899e5c17a9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts @@ -0,0 +1,109 @@ +// Startup no longer waits for the lease check, so a write can reach a chat whose lease the previous +// app run left unadjudicated. Every write waits for that chat's check (or runs it) before it acts, +// so none is admitted against the old lease and none starts a second agent. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { hostTestOperationId } from './structured-agent-session-host-test-data' +import { + createStartupRig, + STARTUP_CALLER, + type StartupRig +} from './structured-agent-session-startup-listing-test-rig' + +let rig: StartupRig +// A failed assertion must not leave teardown waiting on a held probe. +let releaseProbe = (): void => undefined + +beforeEach(async () => { + rig = await createStartupRig() +}) + +afterEach(async () => { + releaseProbe() + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +/** Boots over a crashed run and starts the startup lease check with its owner probe held. */ +async function bootWithSlowLeaseCheck() { + await rig.chat('session-a', { message: 'before the restart' }) + await rig.crash() + const probe = Promise.withResolvers() + const probing = Promise.withResolvers() + releaseProbe = probe.resolve + rig.probeOwner.mockImplementation(async () => { + probing.resolve() + await probe.promise + return { outcome: 'pid-absent' } + }) + const host = await rig.boot() + // What startup does now: start the check, and wait for nothing. + const startupCheck = host.reconcileRestartLeases() + await probing.promise + // Every admission, with the lease it was admitted against. + const admittedUnreconciled: boolean[] = [] + const admit = rig.store.admitMutationOperation + vi.spyOn(rig.store, 'admitMutationOperation').mockImplementation(async (args) => { + const admitted = await admit(args) + admittedUnreconciled.push(admitted?.record.lease.unreconciled ?? true) + return admitted + }) + return { host, probe, startupCheck, admittedUnreconciled } +} + +/** Long enough for an unheld write to open its chat and be admitted. */ +function settleTasks(): Promise { + return new Promise((resolve) => setTimeout(resolve, 200)) +} + +describe('a write during a slow startup lease check', () => { + it('lands a send after the chat’s check, and starts one agent (REQUIRED)', async () => { + const { probe, startupCheck, admittedUnreconciled } = await bootWithSlowLeaseCheck() + + let answered = false + const sent = rig.send('session-a', 'after the restart').then((result) => { + answered = true + return result + }) + await settleTasks() + expect(answered).toBe(false) + expect(admittedUnreconciled).toEqual([]) + + probe.resolve() + await expect(sent).resolves.toMatchObject({ ok: true }) + await startupCheck + await vi.waitFor(() => expect(rig.dispatch).toHaveBeenCalledOnce()) + + expect(admittedUnreconciled).toEqual([false]) + expect(rig.acquire).toHaveBeenCalledOnce() + }) + + it('holds a Stop until the chat’s check has run', async () => { + const { host, probe, startupCheck, admittedUnreconciled } = await bootWithSlowLeaseCheck() + + const fields = {} + const stopped = host.cancel(STARTUP_CALLER, { + envelope: { + sessionId: 'session-a', + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: 'session-a', + fields + }) + } + }) + await settleTasks() + expect(admittedUnreconciled).toEqual([]) + + probe.resolve() + await expect(stopped).resolves.toMatchObject({ ok: true }) + await startupCheck + expect(admittedUnreconciled).toEqual([false]) + expect(rig.acquire).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index c0d4c596ffae..6ea17700c7a3 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -252,16 +252,27 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR return this.structuredAgentSessionTabRestorePromise } - prepareStructuredAgentSessionStartupRestoration(): Promise { + /** Startup's structured step: the host and PTY inventory, awaited; the lease check, started and + * never awaited. Neither the tab list nor paint needs leases, and every write to a chat whose + * lease is unchecked waits for its own check (`serializeAfterLeaseCheck`). */ + async prepareStructuredAgentSessionStartupRestoration(): Promise { + await this.ensureStructuredAgentSessionInventory() + if (this.hasPersistedStructuredAgentSessionStore()) { + // Never rejects: a failure goes to the host's startup-reconcile report. + void getStructuredAgentSessionHost()?.reconcileRestartLeases() + } + } + + protected ensureStructuredAgentSessionInventory(): Promise { this.structuredAgentSessionStartupRestorePromise ??= - this.prepareStructuredAgentSessionStartupRestorationOnce().catch((error) => { + this.ensureStructuredAgentSessionInventoryOnce().catch((error) => { this.structuredAgentSessionStartupRestorePromise = null throw error }) return this.structuredAgentSessionStartupRestorePromise } - protected async prepareStructuredAgentSessionStartupRestorationOnce(): Promise { + private async ensureStructuredAgentSessionInventoryOnce(): Promise { if (!this.hasPersistedStructuredAgentSessionStore()) { return } @@ -271,7 +282,6 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR this.ensureStructuredAgentSessionHost() ) await this.refreshMobileSessionPtyRecords() - await getStructuredAgentSessionHost()?.reconcileRestartLeases() } protected hasPersistedStructuredAgentSessionStore(): boolean { diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 8d29a0d360b9..90a45ed48a9e 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -54,12 +54,11 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu typeof host?.getPersistedVisibleSessionTabIndex === 'function' ? host.getPersistedVisibleSessionTabIndex() : { present: false, sessionIds: [] } - const profileIds = collectSavedStructuredAgentSessionIds( - this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null - ) - await host?.restoreReadableSessions( - persistedVisibleIndex.present ? persistedVisibleIndex.sessionIds : profileIds - ) + const listedIds = persistedVisibleIndex.present + ? persistedVisibleIndex.sessionIds + : collectSavedStructuredAgentSessionIds( + this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + ) for (const worktreeId of this.getKnownWorkspaceSessionWorktreeIds()) { this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { allowAttachedWindow: true, @@ -70,7 +69,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const replacement of host?.conversationReplacements?.() ?? []) { await this.replaceStructuredAgentSessionTab(replacement) } - for (const session of host?.listSessionTabs() ?? []) { + for (const session of host?.listSessionTabs(listedIds) ?? []) { if (session.agent !== 'codex' && session.agent !== 'claude') { continue } @@ -86,6 +85,11 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu notify: false }) } + // After the tabs are out, so no chat's history holds the list or paint; each chat's status row + // arrives as its history opens. One chat's failure leaves the others and the list alone. + void host?.restoreReadableSessions(listedIds).catch((error: unknown) => { + console.warn('[structured-agent-session] restoring chat history after listing failed', error) + }) const wasUnverifiable = this.structuredAgentSessionInventoryUnverifiable // No host means no one can say which chats exist; with none on disk, empty is the answer. this.structuredAgentSessionInventoryUnverifiable = diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 1d8105d07f81..351c3068aaf6 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -51,7 +51,7 @@ describe('structured session cold restoration', () => { expect(restoreReadableSessions).not.toHaveBeenCalled() }) - it('loads records, inventories PTYs, restores ownership, then projects tabs exactly once', async () => { + it('loads records, inventories PTYs, starts the lease check, projects tabs, then restores history, once', async () => { const runtime = new OrcaRuntimeService() const hydrate = vi.fn() const refresh = vi.fn(async () => new Set()) @@ -102,8 +102,9 @@ describe('structured session cold restoration', () => { expect(reconcileRestartLeases.mock.invocationCallOrder[0]).toBeLessThan( restoreReadableSessions.mock.invocationCallOrder[0] ?? Infinity ) - expect(restoreReadableSessions.mock.invocationCallOrder[0]).toBeLessThan( - hydrate.mock.invocationCallOrder[0] ?? Infinity + // Tabs are projected first; history opens after, so no chat's history holds the list. + expect(hydrate.mock.invocationCallOrder[0]).toBeLessThan( + restoreReadableSessions.mock.invocationCallOrder[0] ?? Infinity ) }) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts index 4aca4d0420a0..09ebcdf3e617 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts @@ -19,6 +19,7 @@ import { workerTerminalLeaseIsCurrent } from './worker-terminal-release-lease' import { resolveStructuredWorkerForDispatch } from '../../orchestration-structured-worker-lifecycle' import { stopStructuredWorkerForRelease } from './structured-worker-release-stop' import { isStructuredWorkerHandle } from '../../../../structured-worker-identity' +import { getStructuredAgentSessionHost } from '../../../../../native-chat/agent-session-wire/structured-agent-session-registry' export { archiveSummary, @@ -95,7 +96,9 @@ async function completeWorkerTerminalReleaseOnce( if (isStructuredWorkerHandle(resource.terminal_handle)) { // Observation and archive capture both read the structured host, and after a restart nothing // has installed it yet — the startup recovery reconciler runs exactly this path. Installing it - // here is what lets the release see the session instead of reporting it unreadable. + // here is what lets the release see the session instead of reporting it unreadable. Startup no + // longer waits for the lease check either, and an unchecked worker lease from the previous run + // reads as live, so the release runs that check first (it never rejects; failures are reported). // // NOT yet handled, and deliberately follow-up: rebinding a restarted runtime to a structured // worker's redrive subscription. Until that exists, a worker that survives a restart has its @@ -107,6 +110,7 @@ async function completeWorkerTerminalReleaseOnce( error ) }) + await getStructuredAgentSessionHost()?.reconcileRestartLeases() } const worker = db.getWorkerDispatch(dispatchId) if (!worker || worker.agent_terminal_handle !== resource.terminal_handle) { diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts new file mode 100644 index 000000000000..7972b485f7f1 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts @@ -0,0 +1,62 @@ +// After a restart the release is the first thing to read a structured worker, and startup no longer +// waits for the lease check. The release runs that check before it observes the worker (T6). + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { setStructuredAgentSessionHost } from '../../../../../native-chat/agent-session-wire/structured-agent-session-registry' +import { mintStructuredWorkerHandle } from '../../../../structured-worker-identity' + +const { order } = vi.hoisted(() => { + const order: string[] = [] + return { order } +}) + +vi.mock('./worker-observation', () => ({ + inspectWorkerTerminal: vi.fn(async () => { + order.push('observe') + return { status: 'identity_changed' } + }) +})) + +const { completeWorkerTerminalRelease } = await import('./worker-release-completion') + +afterEach(() => { + order.length = 0 + setStructuredAgentSessionHost(null) +}) + +describe('structured worker release after a restart', () => { + it('reads the worker only once the restart reconcile has settled its lease', async () => { + const handle = mintStructuredWorkerHandle() + const runtime = { + ensureStructuredAgentSessionHost: vi.fn(async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the release reads only the reconcile stubbed here. + setStructuredAgentSessionHost({ + reconcileRestartLeases: async () => { + // A reconcile that probes processes settles after a macrotask, not a microtask. + await new Promise((resolve) => setTimeout(resolve, 0)) + order.push('reconcile') + } + } as never) + }) + } + const db = { + getWorkerDispatch: () => ({ agent_terminal_handle: handle }), + revertWorkerTerminalReleaseToRetained: () => ({ archive_source: null, archive_status: null }), + recordWorkerTerminalRecoveryAttempt: vi.fn() + } + + const receipt = await completeWorkerTerminalRelease({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this path reads only the runtime member stubbed above. + runtime: runtime as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retained path reads only the dispatch rows stubbed above. + db: db as never, + dispatchId: 'dispatch-1', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retained path reads only the id and handle. + resource: { id: 'resource-1', terminal_handle: handle } as never, + mode: 'recovery' + }) + + expect(receipt.state).toBe('retained') + expect(order).toEqual(['reconcile', 'observe']) + }) +}) diff --git a/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts b/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts index b48dccf52a39..0945d32834f0 100644 --- a/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts +++ b/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts @@ -43,9 +43,11 @@ function installHost(options: { closeThrows?: boolean; lease?: Record undefined, hasSession: () => attached, hold: async () => {}, release: () => {}, diff --git a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts index 4b444eef70e1..d8f85a11ee66 100644 --- a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts +++ b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts @@ -83,7 +83,8 @@ it('finishes startup over records a newer Orca wrote, reports it, and writes not // What the renderer's startup awaits through `app:prepareTerminalStartupRestoration`. await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).resolves.toBeUndefined() - expect(onError).toHaveBeenCalledOnce() + // Startup does not wait for the check; its failure is reported when it lands. + await vi.waitFor(() => expect(onError).toHaveBeenCalledOnce()) expect(onError).toHaveBeenCalledWith({ scope: 'structured-agent-session-startup-reconcile', error: expect.objectContaining({ message: 'agent_session_legacy_required' }) @@ -103,8 +104,10 @@ it('logs the failure when the host has no error sink', async () => { startupRuntime().prepareStructuredAgentSessionStartupRestoration() ).resolves.toBeUndefined() - expect(warn).toHaveBeenCalledWith( - '[structured-agent-session] reconciling chat leases at startup failed', - expect.objectContaining({ message: 'agent_session_legacy_required' }) + await vi.waitFor(() => + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] reconciling chat leases at startup failed', + expect.objectContaining({ message: 'agent_session_legacy_required' }) + ) ) }) diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts new file mode 100644 index 000000000000..669732c321e6 --- /dev/null +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -0,0 +1,245 @@ +// The chat tab list a client asks for at startup answers from records and the tab table. It waits +// for no chat's history and for no lease check; each chat's history opens after the answer. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../shared/protocol-version' +import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' +import { closeTestJournalHostDatabases } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { + createStartupRig, + latestStatus, + type StartupRig +} from '../native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig' +import { OrcaRuntimeService } from './orca-runtime' +import { RpcDispatcher } from './rpc/dispatcher' +import { SESSION_TAB_METHODS } from './rpc/methods/session-tabs' + +type ListingInternals = { + store: { getWorkspaceSession: () => unknown } + getClientSettings(): { experimentalStructuredNativeChat: boolean } + supportsAuthoritativeSessionTabsInventory(): boolean + hasPersistedStructuredAgentSessionStore(): boolean + getKnownWorkspaceSessionWorktreeIds(): Set + hydrateHeadlessMobileSessionTabsFromWorkspaceSession(): Set + refreshMobileSessionPtyRecords(): Promise | null> + ensureStructuredAgentSessionHost(): Promise +} + +const CONTEXT = { + clientKind: 'runtime' as const, + clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] +} + +let rig: StartupRig +// Anything a test holds is let go before teardown, so a failed assertion cannot hang it. +let releaseHeld = (): void => undefined + +beforeEach(async () => { + rig = await createStartupRig() +}) + +afterEach(async () => { + releaseHeld() + setStructuredAgentSessionHost(null) + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +/** A restarted runtime whose structured host is the rig's current one. */ +function restartedRuntime(workspaceSession: unknown = null) { + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as ListingInternals + internal.store = { getWorkspaceSession: () => workspaceSession } + internal.getClientSettings = () => ({ experimentalStructuredNativeChat: true }) + // The PTY census behind an authoritative inventory is not what this pins. + internal.supportsAuthoritativeSessionTabsInventory = () => false + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.getKnownWorkspaceSessionWorktreeIds = () => new Set() + internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() + internal.refreshMobileSessionPtyRecords = async () => new Set() + internal.ensureStructuredAgentSessionHost = async () => { + setStructuredAgentSessionHost(rig.host) + } + const dispatcher = new RpcDispatcher({ runtime, methods: SESSION_TAB_METHODS }) + const listAll = async (): Promise => { + const response = await dispatcher.dispatch( + { id: 'list-all', authToken: 'tok', method: 'session.tabs.listAll' }, + CONTEXT + ) + if (!response.ok) { + throw new Error(`listAll refused: ${JSON.stringify(response.error)}`) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: listAll answers with the inventory shape its handler returns. + const { snapshots } = response.result as { snapshots: RuntimeMobileSessionTabsResult[] } + return snapshots.flatMap((snapshot) => + snapshot.tabs.flatMap((tab) => (tab.type === 'agent-session' ? [tab.sessionId] : [])) + ) + } + return { runtime, listAll } +} + +/** Resolves with the answer, or with 'still waiting' once `ms` pass without one. */ +function within(promise: Promise, ms = 2_000): Promise { + return Promise.race([ + promise, + new Promise<'still waiting'>((resolve) => setTimeout(() => resolve('still waiting'), ms)) + ]) +} + +describe('listing chat tabs at startup', () => { + it('answers before any chat opens, then opens each one (T1)', async () => { + const ids = ['session-1', 'session-2', 'session-3'] + for (const sessionId of ids) { + await rig.chat(sessionId, { message: sessionId }) + } + await rig.crash() + await rig.boot() + const opens = Promise.withResolvers() + releaseHeld = opens.resolve + let opened = 0 + rig.historyFilePath.mockImplementation(async () => { + await opens.promise + opened += 1 + return null + }) + const { listAll } = restartedRuntime() + + expect(await within(listAll())).toEqual(ids) + expect(opened).toBe(0) + + opens.resolve() + await vi.waitFor(() => ids.forEach((id) => expect(rig.host.hasSession(id)).toBe(true))) + expect(opened).toBe(3) + }) + + it('answers while the startup lease check never finishes (T2)', async () => { + await rig.chat('session-1', { message: 'one' }) + await rig.chat('session-2', { message: 'two' }) + await rig.crash() + const probe = Promise.withResolvers() + releaseHeld = probe.resolve + rig.probeOwner.mockImplementation(async () => { + await probe.promise + return { outcome: 'pid-absent' } + }) + await rig.boot() + const { runtime, listAll } = restartedRuntime() + + // What the renderer's startup awaits through `app:prepareTerminalStartupRestoration`. + expect(await within(runtime.prepareStructuredAgentSessionStartupRestoration())).toBe(undefined) + expect(await within(listAll())).toEqual(['session-1', 'session-2']) + expect(rig.probeOwner).toHaveBeenCalled() + probe.resolve() + await rig.host.reconcileRestartLeases() + }) + + it('answers when the startup lease check fails (T2)', async () => { + await rig.chat('session-1', { message: 'one' }) + await rig.crash() + rig.probeOwner.mockRejectedValue(new Error('probe unavailable')) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const onStartupReconcileFailure = vi.fn() + await rig.boot({ onStartupReconcileFailure }) + const { runtime, listAll } = restartedRuntime() + + await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).resolves.toBe(undefined) + expect(await within(listAll())).toEqual(['session-1']) + await vi.waitFor(() => + expect(onStartupReconcileFailure).toHaveBeenCalledWith( + expect.objectContaining({ message: 'probe unavailable' }) + ) + ) + }) + + it('lists every chat when one fails to open, and the others still get status rows (T5)', async () => { + const ids = ['session-1', 'session-2', 'session-3'] + for (const sessionId of ids) { + await rig.chat(sessionId, { message: `asked ${sessionId}` }) + } + await rig.crash() + await rig.boot() + rig.historyFilePath.mockImplementation(async (sessionId) => { + if (sessionId === 'session-2') { + throw new Error('EACCES: permission denied') + } + return null + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const restore = vi.spyOn(rig.host, 'restoreReadableSessions') + const { listAll } = restartedRuntime() + + expect(await within(listAll())).toEqual(ids) + await vi.waitFor(() => { + expect(latestStatus(rig, 'session-1')).toMatchObject({ latestPrompt: 'asked session-1' }) + expect(latestStatus(rig, 'session-3')).toMatchObject({ latestPrompt: 'asked session-3' }) + }) + await vi.waitFor(() => + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] restoring chat history after listing failed', + expect.objectContaining({ message: 'EACCES: permission denied' }) + ) + ) + expect(await listAll()).toEqual(ids) + expect(restore).toHaveBeenCalledOnce() + expect( + warn.mock.calls.filter(([message]) => String(message).includes('after listing')) + ).toHaveLength(1) + }) + + it('lists in tab-table order, not the order the chats were created (T7)', async () => { + for (const sessionId of ['session-c', 'session-a', 'session-b']) { + await rig.chat(sessionId, { listed: false }) + } + for (const sessionId of ['session-b', 'session-c', 'session-a']) { + await rig.store.setSessionTabVisibility(sessionId, true) + } + await rig.crash() + await rig.boot() + const { listAll } = restartedRuntime() + + expect(await within(listAll())).toEqual(['session-b', 'session-c', 'session-a']) + }) + + it('lists in the saved window order when the profile has no tab table (T7)', async () => { + for (const sessionId of ['session-c', 'session-a', 'session-b']) { + await rig.chat(sessionId, { listed: false }) + } + await rig.crash() + const host = await rig.boot() + vi.spyOn(host, 'getPersistedVisibleSessionTabIndex').mockReturnValue({ + present: false, + sessionIds: [] + }) + const savedTab = (sessionId: string, sortOrder: number) => ({ + id: `agent-session:${sessionId}`, + entityId: sessionId, + groupId: 'group-1', + worktreeId: 'workspace-1', + contentType: 'agent-session', + label: 'Codex Chat', + customLabel: null, + color: null, + sortOrder, + createdAt: 1 + }) + const { listAll } = restartedRuntime({ + activeRepoId: null, + activeWorktreeId: 'workspace-1', + activeTabId: null, + tabsByWorktree: {}, + terminalLayoutsByTabId: {}, + unifiedTabs: { + 'workspace-1': [ + savedTab('session-a', 0), + savedTab('session-b', 1), + savedTab('session-c', 2) + ] + } + }) + + expect(await within(listAll())).toEqual(['session-a', 'session-b', 'session-c']) + }) +}) From b23eb280daa9317226941d24a883caeb895bf28b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:01:18 -0700 Subject: [PATCH 004/107] fix(native-chat): keep every record-store write off the startup chat read path Round-2 review found two more writes on the startup chat restore that could still fail it and put the app into "Session restore failed": republishing a /clear replacement recorded its tab visibility strictly, and resolving a chat's recovery rethrew its store error. The restore also paid one lock wait per tab and per batch of chats while the lock stayed held. The restore now derives tabs from state it already holds: - publishStructuredAgentSessionTab splits into the strict write and projectStructuredAgentSessionTab, which only updates the runtime's snapshot. The restore and /clear replacements only project: a saved tab index already lists every restored chat, and a /clear moves the tab in the same write that commits it. visibilityWriteMayFail is gone. - Chats a legacy profile restores that the index does not list are recorded in one best-effort transaction (store.showSessionTabs), so a failure leaves the index absent to seed again rather than partial. - The read restore's recovery resolution is caught and reported through onLeaseReconcileFailure, deduplicated with the reconcile's reports. - Once lease bookkeeping fails in a restore pass, the rest of that pass skips it, so a held lock costs one wait for the startup reconcile and one for the restore, however many chats are open. User actions (create, reveal, attach, send, the /clear commit) keep their strict writes. --- .../structured-agent-session-host-types.ts | 5 +- .../structured-agent-session-host.ts | 1 + ...ed-agent-session-readable-restorer.test.ts | 2 +- ...uctured-agent-session-restart-reconcile.ts | 39 +- ...ured-agent-session-restart-restore.test.ts | 7 +- ...tructured-agent-session-restart-restore.ts | 26 +- ...ent-session-restore-without-import.test.ts | 2 +- .../structured-agent-session-reveal.ts | 28 +- ...-session-startup-reconcile-failure.test.ts | 35 +- .../runtime/agent-session-record-store.ts | 8 +- src/main/runtime/agent-session-tab-table.ts | 12 + ...tore-structured-agent-session-tabs-once.ts | 65 ++-- ...runtime-structured-session-restore.test.ts | 31 +- .../rpc/methods/structured-agent-session.ts | 2 +- ...ed-agent-session-startup-reconcile.test.ts | 156 +------- ...-agent-session-startup-tab-restore.test.ts | 351 ++++++++++++++++++ ...structured-agent-session-tab-index-seed.ts | 35 ++ 17 files changed, 593 insertions(+), 212 deletions(-) create mode 100644 src/main/runtime/structured-agent-session-startup-tab-restore.test.ts create mode 100644 src/main/runtime/structured-agent-session-tab-index-seed.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index e2e8e07d4fb3..f69bc3fb06aa 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -120,8 +120,9 @@ export type StructuredAgentSessionHostDeps = { /** Whether an orchestration dispatch still owns this session's worker; absent answers no. */ hasOpenDispatch?: (record: AgentSessionRecord) => boolean onEventSinkError?: (input: { sessionId: string; error: unknown }) => void - /** A lease reconcile run for startup or a read that refused or threw, once per distinct failure. - * Startup and the read carry on: the next attach or send reconciles every lease again. */ + /** Lease bookkeeping run for startup or a read (the reconcile, or resolving a chat's recovery) + * that refused or threw, once per distinct failure. Startup and the read carry on: the next + * attach or send reconciles and resolves recovery again before it acts. */ onLeaseReconcileFailure?: (failure: unknown) => void /** Every status projection this host publishes. `replay` marks a re-projection of state the host * already knew (restore, an arriving subscriber) rather than a fresh journal edge. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 78bc7de155ef..406f105a7c75 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -215,6 +215,7 @@ export class StructuredAgentSessionHost { listSessionTabs = () => sessionTabs.listStructuredAgentSessionTabs(this.sessions) getPersistedVisibleSessionTabIndex = () => this.deps.store.getVisibleSessionTabIndex() getSessionTabId = (sessionId: string): string | null => this.deps.store.getSessionTabId(sessionId) + showSessionTabs = (sessionIds: readonly string[]) => this.deps.store.showSessionTabs(sessionIds) setSessionTabVisibility = async ( sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index 240882f17564..09cab618ab9a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -40,7 +40,7 @@ describe('StructuredAgentSessionReadableRestorer', () => { }, supportsRecord: () => true, reconcile: async () => true, - resolveRecovery: async () => undefined, + resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, onReadable: () => undefined diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index c9cba45aa59d..be2d0b6c1513 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -45,32 +45,51 @@ export function createRestartReconciler(deps: { } } +/** Reports each distinct failure once, until `clear` says the bookkeeping settled again. Every read + * re-runs a failing reconcile, so without this a failing store would log on every chat list. */ +export type ReaderBookkeepingFailures = { + report: (failure: unknown) => void + clear: () => void +} + +export function reportEachFailureOnce( + onFailure: ((failure: unknown) => void) | undefined +): ReaderBookkeepingFailures { + let reported: string | null = null + return { + report: (failure) => { + const key = failureKey(failure) + if (key !== reported) { + reported = key + onFailure?.(failure) + } + }, + clear: () => { + reported = null + } + } +} + /** The reconcile a reader runs, at startup and before each restored read: it never throws, since * an unreconciled lease grants no writer and the next send reconciles again before it acts. - * Answers whether every lease is settled; a failure is reported once until a reconcile settles. */ + * Answers whether every lease is settled. */ export function createReaderReconcile( reconcile: (sessionId: string) => Promise, - onFailure: ((failure: unknown) => void) | undefined + failures: ReaderBookkeepingFailures ): (sessionId: string) => Promise { - let reported: string | null = null return async (sessionId) => { let failure: unknown try { const refusal = await reconcile(sessionId) if (!refusal) { - reported = null + failures.clear() return true } failure = refusal } catch (error) { failure = error } - // Every read re-runs a failing reconcile, so each distinct failure is reported once. - const key = failureKey(failure) - if (key !== reported) { - reported = key - onFailure?.(failure) - } + failures.report(failure) return false } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 3bf000f9e409..c7d6b21ea1b0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -63,7 +63,7 @@ describe('restart journal restoration', () => { openDeps: NO_OPEN_DEPS, records, reconcile: async () => true, - resolveRecovery: async () => undefined, + resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, onReadable: () => undefined @@ -104,7 +104,7 @@ describe('restart journal restoration', () => { openDeps: NO_OPEN_DEPS, records, reconcile: async () => true, - resolveRecovery: async () => undefined, + resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, onReadable: () => undefined @@ -153,6 +153,7 @@ describe('restart journal restoration', () => { reconcile: async () => true, resolveRecovery: async () => { calls.push('resolveRecovery') + return true }, serialize: async (_sessionId, task) => task(), hasSession: () => false, @@ -175,7 +176,7 @@ describe('restart journal restoration', () => { records: [{ sessionId: 'session-1' } as AgentSessionRecord], openDeps: NO_OPEN_DEPS, reconcile: async () => true, - resolveRecovery: async () => undefined, + resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => true, onReadable: () => undefined diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 02561594caa8..facc67a86a92 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -27,9 +27,11 @@ export type StructuredAgentSessionReadRestoreDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { store: Pick } - /** Whether every lease is settled. Never throws: a read grants no writer, so bookkeeping can't block it. */ + // Lease bookkeeping. Neither throws: a read grants no writer, so bookkeeping must not block it. + /** Whether every lease is settled. */ reconcile: (sessionId: string) => Promise - resolveRecovery: (sessionId: string) => Promise + /** False when its store write failed; the next attach or send resolves it again. */ + resolveRecovery: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean onReadable: ( @@ -41,12 +43,10 @@ export type StructuredAgentSessionReadRestoreDeps = { /** One session's share of the restart restore. Startup maps this over every supported record. */ async function restoreOneStructuredAgentSessionRead( input: StructuredAgentSessionReadRestoreDeps, - sessionId: string + sessionId: string, + settleLeases: (sessionId: string) => Promise ): Promise { - if (await input.reconcile(sessionId)) { - // A session latched in recovery exits here at startup, without waiting for a client. - await input.resolveRecovery(sessionId) - } + await settleLeases(sessionId) await input.serialize(sessionId, () => restoreOneStructuredAgentSessionReadUnderSerialize(input, sessionId) ) @@ -72,9 +72,19 @@ async function restoreOneStructuredAgentSessionReadUnderSerialize( export async function restoreStructuredAgentSessionsOnRestart( input: StructuredAgentSessionReadRestoreDeps & { records: AgentSessionRecord[] } ): Promise { + // Once bookkeeping fails in this pass, trying again per chat only waits on the same store again. + let bookkeepingFailed = false + const settleLeases = async (sessionId: string): Promise => { + if (bookkeepingFailed) { + return + } + // A session latched in recovery exits here at startup, without waiting for a client. + const settled = (await input.reconcile(sessionId)) && (await input.resolveRecovery(sessionId)) + bookkeepingFailed ||= !settled + } await mapWithConcurrency(input.records, JOURNAL_RESTORE_CONCURRENCY, async ({ sessionId }) => { // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. await yieldToEventLoop() - await restoreOneStructuredAgentSessionRead(input, sessionId) + await restoreOneStructuredAgentSessionRead(input, sessionId, settleLeases) }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts index 1308dbfe4a83..3290ba05f0de 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts @@ -233,7 +233,7 @@ async function restore(sessionIds: readonly string[]) { openDeps: deps, records: sessionIds.map(recordFor), reconcile: async () => true, - resolveRecovery: async () => undefined, + resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: (sessionId) => sessions.has(sessionId), onReadable: (sessionId, opened) => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 96b0abd2b113..262c9e18d3c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -15,7 +15,10 @@ import { agentSessionRefusalError } from '../../../shared/agent-session-wire-ref import { adapterSupportsRecord } from './structured-agent-session-provider-support' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' -import { createReaderReconcile } from './structured-agent-session-restart-reconcile' +import { + createReaderReconcile, + reportEachFailureOnce +} from './structured-agent-session-restart-reconcile' import type { StructuredAgentSessionHostDeps, StructuredAgentSessionReveal @@ -54,23 +57,36 @@ export async function revealStructuredAgentSession( } /** The host's startup readable-restore sweep: reconcile, resolve, then open each chat's journal. - * Its reconcile is a reader's, which never fails a read or startup; startup runs the same one. */ + * Its lease bookkeeping is a reader's, which never fails a read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], - 'openDeps' | 'supportsRecord' | 'reconcile' - > & { reconcileLeases: (sessionId: string) => Promise } + 'openDeps' | 'supportsRecord' | 'reconcile' | 'resolveRecovery' + > & { + reconcileLeases: (sessionId: string) => Promise + resolveRecovery: (sessionId: string) => Promise + } ): { reconcileRestartLeases: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise } { - const { reconcileLeases, ...rest } = wiring - const reconcile = createReaderReconcile(reconcileLeases, deps.onLeaseReconcileFailure) + const { reconcileLeases, resolveRecovery, ...rest } = wiring + const failures = reportEachFailureOnce(deps.onLeaseReconcileFailure) + const reconcile = createReaderReconcile(reconcileLeases, failures) const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: deps, supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), reconcile, + // The next attach or send resolves recovery again, strictly, before it acts. + resolveRecovery: (sessionId) => + resolveRecovery(sessionId).then( + () => true, + (error: unknown) => { + failures.report(error) + return false + } + ), ...rest }) const gate = new StructuredAgentSessionRestartRestoreGate() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts index 5e1c8a95868c..8033843cd41e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts @@ -11,7 +11,10 @@ import { AGENT_SESSION_STORE_SCHEMA_VERSION, agentSessionStorePath } from '../../runtime/agent-session-record-store-file' -import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + StructuredAgentSessionHost, + type StructuredAgentSessionHostDeps +} from './structured-agent-session-host' import { adapter, attach, @@ -53,7 +56,12 @@ afterEach(async () => { }) /** A host with one chat, relaunched over a copy of its files; `rewrite` edits the copied store. */ -async function relaunch(rewrite?: (json: Record) => void) { +async function relaunch( + rewrite?: (json: Record) => void, + probeOwner: StructuredAgentSessionHostDeps['probeOwner'] = async () => ({ + outcome: 'pid-absent' + }) +) { const dying = hostTestState() await attach() // An empty renewal queues behind every record write, so they are on disk. @@ -80,7 +88,10 @@ async function relaunch(rewrite?: (json: Record) => void) { journalDatabase: openTestJournalHostDatabase(relaunched), claimKeyId: 'key-1', mintSpawnToken: () => 'spawn-next', - probeOwner: async () => ({ outcome: 'pid-absent' }), + probeOwner, + stopOwnerProcess: () => { + throw new Error('a read must not stop an owner') + }, now: () => NOW, onLeaseReconcileFailure }) @@ -168,3 +179,21 @@ it('restores a chat for reading from a store a newer Orca wrote', async () => { expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() expect(await readFile(path)).toEqual(bytes) }) + +// A chat whose owner could not be proven gone is left recovering; resolving that is the writer's. +it('restores a chat for reading when resolving its recovery cannot write the store', async () => { + const { host, store, onLeaseReconcileFailure } = await relaunch(undefined, async () => ({ + outcome: 'indeterminate', + reason: 'probe' + })) + await host.reconcileRestartLeases() + expect(store.getRecord(SESSION)?.lease.handoffStage).toBe('recovering') + lock.failing = true + + await expect(host.restoreReadableSessions([SESSION])).resolves.toBeUndefined() + + expect(host.hasSession(SESSION)).toBe(true) + expect(store.getRecord(SESSION)?.lease.handoffStage).toBe('recovering') + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() + expect(onLeaseReconcileFailure).toHaveBeenCalledWith(expect.objectContaining({ code: 'ELOCKED' })) +}) diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index 925fa38d8d18..603b5ed02faf 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -66,7 +66,7 @@ import { agentSessionStorePath, type AgentSessionStoreState } from './agent-session-record-store-file' -import { setAgentSessionTabVisibility } from './agent-session-tab-table' +import { setAgentSessionTabVisibility, showAgentSessionTabs } from './agent-session-tab-table' import { loadProtectedAgentSessionStore } from './agent-session-record-store-security' import { AgentSessionStoreTransactionQueue, @@ -146,6 +146,12 @@ export class AgentSessionRecordStore { return this.transact(() => setAgentSessionTabVisibility(this.state, sessionId, visible, tabId)) } + /** Shows each session that still has a record, in one write: an index written part way would + * read as complete at the next launch and drop the rest. */ + showSessionTabs(sessionIds: readonly string[]): Promise { + return this.transact(() => showAgentSessionTabs(this.state, sessionIds)) + } + listByScope(location: AgentSessionExecutionLocation): AgentSessionRecord[] { const scope = agentSessionScopeKey(location) return this.listRecords().filter((record) => agentSessionScopeKey(record.location) === scope) diff --git a/src/main/runtime/agent-session-tab-table.ts b/src/main/runtime/agent-session-tab-table.ts index 6022aac8bf2b..840356cf0cf0 100644 --- a/src/main/runtime/agent-session-tab-table.ts +++ b/src/main/runtime/agent-session-tab-table.ts @@ -131,6 +131,18 @@ export function setAgentSessionTabVisibility( } } +/** Shows each session that still has a record. */ +export function showAgentSessionTabs( + state: AgentSessionStoreState, + sessionIds: readonly string[] +): void { + for (const sessionId of sessionIds) { + if (state.records.has(sessionId)) { + setAgentSessionTabVisibility(state, sessionId, true) + } + } +} + export type PersistedAgentSessionTab = { tabId: string; sessionId: string } export function serializeAgentSessionTabTable(table: AgentSessionTabTable): { diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 476b07820552..1eea896b5dce 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -5,6 +5,7 @@ import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' +import { seedStructuredAgentSessionTabIndex } from './structured-agent-session-tab-index-seed' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { RuntimeMobileSessionAgentTab, @@ -28,7 +29,9 @@ import { parseAppSshPtyId } from '../../shared/ssh-pty-id' import type { PtyProcessInspection } from '../providers/pty-process-inspection' export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRuntimeWithGetStructuredAgentSessionCreateSupport { - async replaceStructuredAgentSessionTab(replacement: ConversationReplacement): Promise { + /** Projects only: a replacement's chat already has its tab in the store, which the /clear commit + * moved in the same write. */ + replaceStructuredAgentSessionTab(replacement: ConversationReplacement): void { const prior = this.mobileSessionTabsByWorktree.get(replacement.workspaceId) const next = prior ? replaceConversationInSnapshot(prior, replacement) : null if (next && next !== prior) { @@ -39,7 +42,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu (tab) => tab.type === 'agent-session' && tab.sessionId === replacement.sessionId ) ) { - await this.publishStructuredAgentSessionTab({ + this.projectStructuredAgentSessionTab({ ...replacement, replacesSessionId: replacement.sourceSessionId, activate: false @@ -57,9 +60,8 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu const profileIds = collectSavedStructuredAgentSessionIds( this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null ) - await host?.restoreReadableSessions( - persistedVisibleIndex.present ? persistedVisibleIndex.sessionIds : profileIds - ) + const targets = persistedVisibleIndex.present ? persistedVisibleIndex.sessionIds : profileIds + await host?.restoreReadableSessions(targets) for (const worktreeId of this.getKnownWorkspaceSessionWorktreeIds()) { this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { allowAttachedWindow: true, @@ -67,25 +69,27 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu }) } this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession() - for (const replacement of host?.conversationReplacements?.() ?? []) { - await this.replaceStructuredAgentSessionTab(replacement) - } - for (const session of host?.listSessionTabs() ?? []) { + const restored = (host?.listSessionTabs() ?? []).flatMap((session) => { if (session.agent !== 'codex' && session.agent !== 'claude') { - continue + return [] } let sessionId = session.sessionId while (sessionId.startsWith('agent-session:')) { sessionId = sessionId.slice('agent-session:'.length) } - await this.publishStructuredAgentSessionTab({ - ...session, - agent: session.agent, - sessionId, - activate: false, - notify: false, - visibilityWriteMayFail: true - }) + return [{ ...session, agent: session.agent, sessionId }] + }) + await seedStructuredAgentSessionTabIndex( + host, + targets, + restored.map((session) => session.sessionId) + ) + // The store already lists every tab projected here, so the restore writes nothing else. + for (const replacement of host?.conversationReplacements?.() ?? []) { + this.replaceStructuredAgentSessionTab(replacement) + } + for (const session of restored) { + this.projectStructuredAgentSessionTab({ ...session, activate: false, notify: false }) } const wasUnverifiable = this.structuredAgentSessionInventoryUnverifiable // No host means no one can say which chats exist; with none on disk, empty is the answer. @@ -106,26 +110,27 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu replacesSessionId?: string /** The host tab id a create reserved; a session that already has a tab keeps its own. */ tabId?: string - /** A restore's republish: a failed visibility write is logged and the tab still publishes, - * since a client drops every unpublished chat tab. */ - visibilityWriteMayFail?: boolean }): Promise { const host = getStructuredAgentSessionHost() if (typeof host?.setSessionTabVisibility === 'function') { - const written = host.setSessionTabVisibility( + await host.setSessionTabVisibility( input.sessionId, true, ...(input.tabId ? [input.tabId] : []) ) - await (input.visibilityWriteMayFail - ? written.catch((error) => - console.warn('[structured-agent-session] recording a restored chat tab failed', { - sessionId: input.sessionId, - error - }) - ) - : written) } + this.projectStructuredAgentSessionTab(input) + } + + /** The runtime's own snapshot of a chat tab. Records nothing: the caller owns the store write. */ + projectStructuredAgentSessionTab(input: { + workspaceId: string + sessionId: string + agent: 'claude' | 'codex' + activate: boolean + notify?: boolean + replacesSessionId?: string + }): void { const existing = this.mobileSessionTabsByWorktree.get(input.workspaceId) const id = `agent-session:${input.sessionId}` if (existing?.tabs.some((tab) => tab.id === id)) { diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 80b8e4f3be65..929ee82b6c89 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -324,7 +324,7 @@ describe('structured session cold restoration', () => { it('publishes restored Claude tabs with the Claude title', async () => { const runtime = new OrcaRuntimeService() - const publish = vi.spyOn(runtime, 'publishStructuredAgentSessionTab') + const project = vi.spyOn(runtime, 'projectStructuredAgentSessionTab') const internal = runtime as unknown as { hasPersistedStructuredAgentSessionStore(): boolean getKnownWorkspaceSessionWorktreeIds(): Set @@ -351,13 +351,12 @@ describe('structured session cold restoration', () => { await runtime.restoreStructuredAgentSessionTabs() - expect(publish).toHaveBeenCalledWith({ + expect(project).toHaveBeenCalledWith({ workspaceId: 'workspace-1', sessionId: 'restored-claude', agent: 'claude', activate: false, - notify: false, - visibilityWriteMayFail: true + notify: false }) const restored = await runtime.listMobileSessionTabs('id:workspace-1') @@ -373,6 +372,30 @@ describe('structured session cold restoration', () => { ) }) + // The /clear commit moved the tab in the store already, so replacing it only projects. + it('replaces a cleared conversation tab without a store write', async () => { + const runtime = new OrcaRuntimeService() + const setSessionTabVisibility = vi.fn(async () => undefined) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: replacing a tab reaches the host only through setSessionTabVisibility, which must stay uncalled. + setStructuredAgentSessionHost({ setSessionTabVisibility } as never) + + runtime.replaceStructuredAgentSessionTab({ + sourceSessionId: 'cleared-session', + sessionId: 'replacement-session', + workspaceId: 'workspace-1', + agent: 'codex' + }) + + const snapshot = await runtime.listMobileSessionTabs('id:workspace-1') + expect(snapshot.tabs).toEqual([ + expect.objectContaining({ + id: 'agent-session:replacement-session', + replacesSessionId: 'cleared-session' + }) + ]) + expect(setSessionTabVisibility).not.toHaveBeenCalled() + }) + it('commits the host close when the renderer already removed the structured tab', async () => { const runtime = new OrcaRuntimeService() runtime.setNotifier({ diff --git a/src/main/runtime/rpc/methods/structured-agent-session.ts b/src/main/runtime/rpc/methods/structured-agent-session.ts index 534d49677793..463772d33375 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.ts @@ -123,7 +123,7 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ .conversationReplacements() .find((entry) => entry.sourceSessionId === params.envelope.sessionId) if (replacement) { - await ctx.runtime.replaceStructuredAgentSessionTab(replacement) + ctx.runtime.replaceStructuredAgentSessionTab(replacement) } await host.close(params.envelope.sessionId) } diff --git a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts index 8fecec36a2ce..33b26f7569bb 100644 --- a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts +++ b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts @@ -5,15 +5,8 @@ import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionRecord } from '../../shared/agent-session-record' -import type { RuntimeMobileSessionTabsSnapshot } from '../../shared/runtime-types' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' -import type * as FileTransactionLock from '../file-transaction-lock' -import { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' -import { openAgentSessionJournal } from '../native-chat/agent-session-journal/journal-store-factory' -import { journalIdentityFor } from '../native-chat/agent-session-wire/structured-agent-session-attach' -import { attachParamsForRecord } from '../native-chat/agent-session-wire/structured-agent-session-conversation-open' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { AGENT_SESSION_STORE_SCHEMA_VERSION, @@ -25,22 +18,6 @@ import { stopStructuredAgentSessionRuntime } from './structured-agent-session-runtime' -const lock = vi.hoisted(() => ({ failing: false })) - -vi.mock('../file-transaction-lock', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - withFileTransactionLock: (...args: Parameters) => - lock.failing - ? // What proper-lockfile throws once its retries give up. - Promise.reject( - Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED' }) - ) - : actual.withFileTransactionLock(...args) - } -}) - let root: string beforeEach(async () => { @@ -48,16 +25,13 @@ beforeEach(async () => { }) afterEach(async () => { - lock.failing = false await stopStructuredAgentSessionRuntime().catch(() => undefined) vi.restoreAllMocks() await rm(root, { recursive: true, force: true }) }) -const PROMPT = 'add a retry' - -/** A profile's record store with one chat in it; one a newer Orca wrote opens read-only here. */ -async function seedStore(options: { newer: boolean; visible?: boolean }) { +/** A profile whose record store a newer Orca wrote, with one chat in it. */ +async function seedNewerStore() { const storeDirectory = join(root, 'agent-sessions') const path = agentSessionStorePath(storeDirectory) const record = agentSessionRecordFixture() @@ -65,61 +39,28 @@ async function seedStore(options: { newer: boolean; visible?: boolean }) { await writeFile( path, JSON.stringify({ - schemaVersion: AGENT_SESSION_STORE_SCHEMA_VERSION + (options.newer ? 1 : 0), + schemaVersion: AGENT_SESSION_STORE_SCHEMA_VERSION + 1, hostId: 'local', records: { [record.sessionId]: record }, operations: {}, retiredClaimKeys: [], - unusableRecords: {}, - ...(options.visible ? { visibleSessionIds: [record.sessionId] } : {}) + unusableRecords: {} }) ) - return { storeDirectory, path, record, sessionId: record.sessionId } + return { storeDirectory, path, sessionId: record.sessionId } } -/** The chat's history as the last run left it: one prompt, accepted. */ -async function seedHistory(record: AgentSessionRecord): Promise { - const fence = record.lease.runtimeFence - const database = JournalHostDatabase.open(root) - const journal = await openAgentSessionJournal({ - identity: journalIdentityFor( - record, - attachParamsForRecord(record, { clientOperationId: 'seed', expectedRuntimeFence: fence }) - ), - database - }) - await journal.appendSubmission({ - clientMessageId: 'client-1', - payloadFingerprint: 'fp-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: PROMPT }] }, - fence, - handoverRecorded: true - }) - await journal.resolveDispatch({ - clientMessageId: 'client-1', - fence, - state: 'accepted', - providerIdentity: null - }) - await journal.close() - database.close() -} - -function startupRuntime( - onError?: (input: { scope: string; error: unknown }) => void, - afterInstall?: () => void -) { +function startupRuntime(onError?: (input: { scope: string; error: unknown }) => void) { const runtime = new OrcaRuntimeService() // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these are the runtime's own protected members; the test roots the host at `root` and stubs the PTY daemon. const internal = runtime as unknown as { hasPersistedStructuredAgentSessionStore(): boolean ensureStructuredAgentSessionHost(): Promise refreshMobileSessionPtyRecords(): Promise | null> - mobileSessionTabsByWorktree: Map } internal.hasPersistedStructuredAgentSessionStore = () => true - internal.ensureStructuredAgentSessionHost = async () => { - const installed = await ensureStructuredAgentSessionHost({ + internal.ensureStructuredAgentSessionHost = () => + ensureStructuredAgentSessionHost({ stateDirectory: root, hostId: 'local', claimKeyId: 'key-1', @@ -128,19 +69,16 @@ function startupRuntime( resolveClaudeAuthPolicy: () => ({ stripAuthEnv: true }), ...(onError ? { onError } : {}) }) - afterInstall?.() - return installed - } internal.refreshMobileSessionPtyRecords = async () => new Set() - return { runtime, publishedTabs: () => internal.mobileSessionTabsByWorktree.get('workspace-1') } + return runtime } it('finishes startup over records a newer Orca wrote, reports it, and writes nothing', async () => { - const { storeDirectory, path, sessionId } = await seedStore({ newer: true }) + const { storeDirectory, path, sessionId } = await seedNewerStore() const bytes = await readFile(path) const files = await readdir(storeDirectory) const onError = vi.fn() - const { runtime } = startupRuntime(onError) + const runtime = startupRuntime(onError) // What the renderer's startup awaits through `app:prepareTerminalStartupRestoration`. await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).resolves.toBeUndefined() @@ -158,11 +96,11 @@ it('finishes startup over records a newer Orca wrote, reports it, and writes not // The desktop installs its host with no error sink, so the failure is logged rather than dropped. it('logs the failure when the host has no error sink', async () => { - await seedStore({ newer: true }) + await seedNewerStore() const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) await expect( - startupRuntime().runtime.prepareStructuredAgentSessionStartupRestoration() + startupRuntime().prepareStructuredAgentSessionStartupRestoration() ).resolves.toBeUndefined() expect(warn).toHaveBeenCalledWith( @@ -170,69 +108,3 @@ it('logs the failure when the host has no error sink', async () => { expect.objectContaining({ message: 'agent_session_legacy_required' }) ) }) - -// With native chat on, the renderer's startup also awaits the chat tab restore (`session.tabs.listAll`), -// which reads every chat whose tab was open at quit. That read must not wait on the lease bookkeeping. -describe('restoring the chat tabs open at quit', () => { - async function expectChatRestored( - runtime: OrcaRuntimeService, - publishedTabs: () => RuntimeMobileSessionTabsSnapshot | undefined, - sessionId: string - ) { - await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() - expect(publishedTabs()?.tabs.map((tab) => tab.id)).toEqual([`agent-session:${sessionId}`]) - const host = getStructuredAgentSessionHost() - expect(JSON.stringify((await host!.journalSnapshot(sessionId)).items)).toContain(PROMPT) - } - - /** The reconcile failure once, though startup and the read both ran it; the tab write per chat. */ - function expectReported( - onError: ReturnType, - warn: ReturnType, - sessionId: string, - failure: Record - ) { - expect(onError).toHaveBeenCalledOnce() - expect(onError).toHaveBeenCalledWith({ - scope: 'structured-agent-session-lease-reconcile', - error: expect.objectContaining(failure) - }) - expect(warn).toHaveBeenCalledWith( - '[structured-agent-session] recording a restored chat tab failed', - { sessionId, error: expect.objectContaining(failure) } - ) - } - - it('lists and reads a chat from records a newer Orca wrote, and writes nothing', async () => { - const { storeDirectory, path, record } = await seedStore({ newer: true, visible: true }) - await seedHistory(record) - const bytes = await readFile(path) - const files = await readdir(storeDirectory) - const onError = vi.fn() - const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const { runtime, publishedTabs } = startupRuntime(onError) - - await expectChatRestored(runtime, publishedTabs, record.sessionId) - - expectReported(onError, warn, record.sessionId, { message: 'agent_session_legacy_required' }) - await stopStructuredAgentSessionRuntime() - expect(await readFile(path)).toEqual(bytes) - expect(await readdir(storeDirectory)).toEqual(files) - }) - - it('lists and reads a chat while the record store lock keeps failing', async () => { - const { record } = await seedStore({ newer: false, visible: true }) - await seedHistory(record) - const onError = vi.fn() - const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) - // Free when the store opens, then held for good: every later write gives up. - const { runtime, publishedTabs } = startupRuntime(onError, () => { - lock.failing = true - }) - - await expectChatRestored(runtime, publishedTabs, record.sessionId) - - expectReported(onError, warn, record.sessionId, { code: 'ELOCKED' }) - lock.failing = false - }) -}) diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts new file mode 100644 index 000000000000..77797a5f531b --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -0,0 +1,351 @@ +// With native chat on, the renderer's startup also awaits the chat tab restore (`session.tabs.listAll`), +// which reads every chat whose tab was open at quit. That read must not wait on record-store +// bookkeeping: with a saved tab index it writes nothing, and a store that cannot be written costs +// a bounded number of lock waits, not one per chat. + +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecord } from '../../shared/agent-session-record' +import type { RuntimeMobileSessionTabsSnapshot } from '../../shared/runtime-types' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../shared/agent-session-record.test-fixture' +import type * as FileTransactionLock from '../file-transaction-lock' +import { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' +import { openAgentSessionJournal } from '../native-chat/agent-session-journal/journal-store-factory' +import { journalIdentityFor } from '../native-chat/agent-session-wire/structured-agent-session-attach' +import { attachParamsForRecord } from '../native-chat/agent-session-wire/structured-agent-session-conversation-open' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { AgentSessionRecordStore } from './agent-session-record-store' +import { + AGENT_SESSION_STORE_SCHEMA_VERSION, + agentSessionStorePath +} from './agent-session-record-store-file' +import { OrcaRuntimeService } from './orca-runtime' +import { + ensureStructuredAgentSessionHost, + stopStructuredAgentSessionRuntime +} from './structured-agent-session-runtime' + +// `failing` refuses every take; `grants` lets that many more through, then refuses. +const lock = vi.hoisted(() => ({ failing: false, grants: Infinity, refused: 0 })) + +vi.mock('../file-transaction-lock', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + withFileTransactionLock: (...args: Parameters) => { + if (lock.failing || lock.grants <= 0) { + lock.refused += 1 + // What proper-lockfile throws once its retries give up, about 3 s later. + return Promise.reject( + Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED' }) + ) + } + lock.grants -= 1 + return actual.withFileTransactionLock(...args) + } + } +}) + +const PROMPT = 'add a retry' +const CHAT_A = 'chat-a-0001' +const CHAT_B = 'chat-b-0002' +const CLEARED = 'chat-s-0003' + +let root: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-tab-restore-')) +}) + +afterEach(async () => { + Object.assign(lock, { failing: false, grants: Infinity, refused: 0 }) + await stopStructuredAgentSessionRuntime().catch(() => undefined) + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) +}) + +/** A released chat, so no startup probe looks for a live owner. */ +function chatRecord( + sessionId: string, + options: { codex?: boolean; clearedInto?: string } = {} +): AgentSessionRecord { + const record = agentSessionRecordFixture( + agentSessionLeaseFixture({ + sessionId, + ownerProcess: null, + reservedSpawnToken: null, + claimStatus: 'released' + }) + ) + const codex = options.codex + ? { + provider: 'codex' as const, + providerHandleChain: record.providerHandleChain.map((link) => ({ + ...link, + handle: { provider: 'codex' as const, threadId: `thread-${sessionId}` } + })), + accountHome: { variable: 'CODEX_HOME' as const, path: join(root, 'codex-home') } + } + : {} + const clear = options.clearedInto + ? { + conversationCommand: { + command: 'clear' as const, + state: 'completed' as const, + phase: 'committed' as const, + operationId: `clear-${sessionId}`, + callerKey: 'client-1', + replacementSessionId: options.clearedInto + } + } + : {} + return { ...record, ...codex, ...clear } +} + +/** Writes the record store; `visible` is the saved tab index, absent on a legacy profile. */ +async function seedStore( + records: AgentSessionRecord[], + options: { newer?: boolean; visible?: string[] } = {} +) { + const storeDirectory = join(root, 'agent-sessions') + const path = agentSessionStorePath(storeDirectory) + await mkdir(storeDirectory, { recursive: true }) + await writeFile( + path, + JSON.stringify({ + schemaVersion: AGENT_SESSION_STORE_SCHEMA_VERSION + (options.newer ? 1 : 0), + hostId: 'local', + records: Object.fromEntries(records.map((record) => [record.sessionId, record])), + operations: {}, + retiredClaimKeys: [], + unusableRecords: {}, + ...(options.visible ? { visibleSessionIds: options.visible } : {}) + }) + ) + return { storeDirectory, path } +} + +/** Each chat's history as the last run left it: one prompt, accepted, so nothing is left to send. */ +async function seedHistory(records: AgentSessionRecord[]): Promise { + const database = JournalHostDatabase.open(root) + for (const record of records) { + const fence = record.lease.runtimeFence + const journal = await openAgentSessionJournal({ + identity: journalIdentityFor( + record, + attachParamsForRecord(record, { clientOperationId: 'seed', expectedRuntimeFence: fence }) + ), + database + }) + await journal.appendSubmission({ + clientMessageId: 'client-1', + payloadFingerprint: 'fp-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: PROMPT }] }, + fence, + handoverRecorded: true + }) + await journal.resolveDispatch({ + clientMessageId: 'client-1', + fence, + state: 'accepted', + providerIdentity: null + }) + await journal.close() + } + database.close() +} + +function startupRuntime(options: { afterInstall?: () => void; profileChats?: string[] } = {}) { + const onError = vi.fn() + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these are the runtime's own protected members; the test roots the host at `root`, stubs the PTY daemon and gives it a profile. + const internal = runtime as unknown as { + hasPersistedStructuredAgentSessionStore(): boolean + ensureStructuredAgentSessionHost(): Promise + refreshMobileSessionPtyRecords(): Promise | null> + mobileSessionTabsByWorktree: Map + store: { getWorkspaceSession: () => unknown } + } + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.ensureStructuredAgentSessionHost = async () => { + const installed = await ensureStructuredAgentSessionHost({ + stateDirectory: root, + hostId: 'local', + claimKeyId: 'key-1', + resolveWorkspacePath: async () => root, + resolveEnvironment: async () => ({}), + resolveClaudeAuthPolicy: () => ({ stripAuthEnv: true }), + onError + }) + options.afterInstall?.() + return installed + } + internal.refreshMobileSessionPtyRecords = async () => new Set() + // A legacy profile's saved chat tabs, which a store with no tab index restores from. + internal.store = { + getWorkspaceSession: () => ({ + activeRepoId: null, + activeWorktreeId: 'workspace-1', + activeTabId: null, + tabsByWorktree: {}, + terminalLayoutsByTabId: {}, + unifiedTabs: { + 'workspace-1': (options.profileChats ?? []).map((sessionId, index) => ({ + id: `agent-session:${sessionId}`, + entityId: sessionId, + groupId: 'group-1', + worktreeId: 'workspace-1', + contentType: 'agent-session', + label: 'Codex Chat', + customLabel: null, + color: null, + sortOrder: index, + createdAt: 1 + })) + } + }) + } + return { + runtime, + onError, + published: () => internal.mobileSessionTabsByWorktree.get('workspace-1')?.tabs ?? [] + } +} + +async function expectHistory(sessionId: string): Promise { + const host = getStructuredAgentSessionHost() + expect(JSON.stringify((await host!.journalSnapshot(sessionId)).items)).toContain(PROMPT) +} + +function spyOnTabWrites() { + return { + visibility: vi.spyOn(AgentSessionRecordStore.prototype, 'setSessionTabVisibility'), + seed: vi.spyOn(AgentSessionRecordStore.prototype, 'showSessionTabs') + } +} + +async function tabIndexOnDisk(path: string): Promise { + return JSON.parse(await readFile(path, 'utf-8')).visibleSessionIds +} + +describe('restoring the chat tabs open at quit', () => { + it.each([ + { store: 'records a newer Orca wrote', newer: true, lockFails: false }, + { store: 'a lock that keeps failing', newer: false, lockFails: true } + ])('lists and reads every chat from $store, and writes nothing', async ({ newer, lockFails }) => { + const records = [ + chatRecord(CHAT_A), + chatRecord(CHAT_B), + chatRecord(CLEARED, { clearedInto: CHAT_A }) + ] + const { storeDirectory, path } = await seedStore(records, { newer, visible: [CHAT_A, CHAT_B] }) + await seedHistory(records.slice(0, 2)) + const bytes = await readFile(path) + const files = await readdir(storeDirectory) + const writes = spyOnTabWrites() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + // Free when the store opens, then held for good. + const { runtime, onError, published } = startupRuntime({ + afterInstall: () => { + lock.failing = lockFails + } + }) + + await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() + + expect(published().map((tab) => tab.id)).toEqual([ + `agent-session:${CHAT_A}`, + `agent-session:${CHAT_B}` + ]) + expect(published()[0]).toMatchObject({ replacesSessionId: CLEARED }) + await expectHistory(CHAT_A) + await expectHistory(CHAT_B) + expect(onError).toHaveBeenCalledOnce() + expect(onError).toHaveBeenCalledWith({ + scope: 'structured-agent-session-lease-reconcile', + error: expect.objectContaining( + newer ? { message: 'agent_session_legacy_required' } : { code: 'ELOCKED' } + ) + }) + expect(writes.visibility).not.toHaveBeenCalled() + expect(writes.seed).not.toHaveBeenCalled() + await stopStructuredAgentSessionRuntime() + if (newer) { + expect(await readFile(path)).toEqual(bytes) + expect(await readdir(storeDirectory)).toEqual(files) + } + }) + + it('costs two lock waits in all while the lock stays held, however many chats are open', async () => { + const chats = Array.from({ length: 8 }, (_, index) => `chat-${index}-000${index}`) + const records = chats.map((sessionId) => chatRecord(sessionId)) + await seedStore(records, { visible: chats }) + await seedHistory(records) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { runtime, published } = startupRuntime({ + afterInstall: () => { + lock.failing = true + } + }) + + await runtime.prepareStructuredAgentSessionStartupRestoration() + await runtime.restoreStructuredAgentSessionTabs() + + expect(published()).toHaveLength(8) + // One for the startup reconcile, one for the restore's; the restore writes no tab. + expect(lock.refused).toBe(2) + }) + + describe('on a legacy profile, with no tab index yet', () => { + const legacyChats = () => [ + chatRecord(CHAT_A, { codex: true }), + chatRecord(CHAT_B, { codex: true }) + ] + + it('records every restored chat in one write', async () => { + const records = legacyChats() + const { path } = await seedStore(records) + await seedHistory(records) + const writes = spyOnTabWrites() + const { runtime, published } = startupRuntime({ profileChats: [CHAT_A, CHAT_B] }) + await runtime.prepareStructuredAgentSessionStartupRestoration() + // One more take, then held: a seed written chat by chat would stop part way. + lock.grants = 1 + + await runtime.restoreStructuredAgentSessionTabs() + + expect(published().map((tab) => tab.id)).toEqual([ + `agent-session:${CHAT_A}`, + `agent-session:${CHAT_B}` + ]) + expect(await tabIndexOnDisk(path)).toEqual([CHAT_A, CHAT_B]) + expect(writes.seed).toHaveBeenCalledOnce() + expect(writes.visibility).not.toHaveBeenCalled() + }) + + it('still lists the chats when that write fails, and leaves the index to seed again', async () => { + const records = legacyChats() + const { path } = await seedStore(records) + await seedHistory(records) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { runtime, published } = startupRuntime({ profileChats: [CHAT_A, CHAT_B] }) + await runtime.prepareStructuredAgentSessionStartupRestoration() + lock.failing = true + + await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() + + expect(published()).toHaveLength(2) + await expectHistory(CHAT_A) + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] recording restored chat tabs failed', + { sessionIds: [CHAT_A, CHAT_B], error: expect.objectContaining({ code: 'ELOCKED' }) } + ) + expect(await tabIndexOnDisk(path)).toBeUndefined() + }) + }) +}) diff --git a/src/main/runtime/structured-agent-session-tab-index-seed.ts b/src/main/runtime/structured-agent-session-tab-index-seed.ts new file mode 100644 index 000000000000..181d67128a1f --- /dev/null +++ b/src/main/runtime/structured-agent-session-tab-index-seed.ts @@ -0,0 +1,35 @@ +// A profile saved before the store kept a tab index restores its chats from the profile's own tabs. +// Those chats are recorded once, all together: an index written part way would read as complete at +// the next launch and drop the rest. With an index present, every restored chat is already listed. + +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' + +type TabIndexHost = Pick< + StructuredAgentSessionHost, + 'getPersistedVisibleSessionTabIndex' | 'showSessionTabs' +> + +/** Best-effort: the tabs publish either way, and an index still absent is seeded again next launch. */ +export async function seedStructuredAgentSessionTabIndex( + host: Partial | null | undefined, + targets: readonly string[], + restored: readonly string[] +): Promise { + if (!host?.getPersistedVisibleSessionTabIndex || !host.showSessionTabs) { + return + } + const listed = new Set(host.getPersistedVisibleSessionTabIndex().sessionIds) + const opened = new Set(restored) + // In the restore's own order, so the seeded tabs keep the order the profile gave them. + const unlisted = [...new Set([...targets, ...restored])].filter( + (sessionId) => opened.has(sessionId) && !listed.has(sessionId) + ) + if (unlisted.length > 0) { + await host.showSessionTabs(unlisted).catch((error: unknown) => + console.warn('[structured-agent-session] recording restored chat tabs failed', { + sessionIds: unlisted, + error + }) + ) + } +} From f3ec1a117020ded62c38c5a7f34dc380a09e7d60 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:14:04 -0700 Subject: [PATCH 005/107] fix(native-chat): start each restore pass from one lease check and stop its bookkeeping at the first failure The restore now runs one reader lease check for the pass and lets each chat re-check and resolve recovery only while the pass is still settled. The first refusal or failed write clears it for the rest of the pass, and every chat is still opened for reading. With another process holding the lock, startup waits on it once in prepare and once in the restore, however many chats are open; a legacy profile waits once more for its tab-index seed. --- ...ured-agent-session-restart-restore.test.ts | 52 ++++++++++++++++ ...tructured-agent-session-restart-restore.ts | 21 ++++--- ...-agent-session-startup-tab-restore.test.ts | 61 +++++++++++++------ 3 files changed, 110 insertions(+), 24 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index c7d6b21ea1b0..0cffa1e32c75 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -165,6 +165,58 @@ describe('restart journal restoration', () => { expect(calls).toEqual(['resolveRecovery', 'open', 'onReadable:restored']) }) + // Each failed bookkeeping call stands for one wait on a held store lock. + describe('once lease bookkeeping fails in a pass', () => { + const records = Array.from( + { length: 8 }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the restore reads only the record's session id here. + (_, index) => ({ sessionId: `session-${index}` }) as AgentSessionRecord + ) + const restore = ( + bookkeeping: Pick< + Parameters[0], + 'reconcile' | 'resolveRecovery' + > + ) => + restoreStructuredAgentSessionsOnRestart({ + openDeps: NO_OPEN_DEPS, + records, + ...bookkeeping, + serialize: async (_sessionId, task) => task(), + hasSession: () => false, + onReadable: () => undefined + }) + + /** A failure that takes a while, as a lock wait does, so the chats open at once overlap it. */ + const slowFailure = async (): Promise => { + await new Promise((resolve) => setTimeout(resolve, 5)) + return false + } + + beforeEach(() => restoreRead.mockResolvedValue(null)) + + it('skips it for every chat when the pass check fails, and still opens them all', async () => { + const reconcile = vi.fn(slowFailure) + const resolveRecovery = vi.fn(async () => true) + + await restore({ reconcile, resolveRecovery }) + + expect(reconcile).toHaveBeenCalledOnce() + expect(resolveRecovery).not.toHaveBeenCalled() + expect(restoreRead).toHaveBeenCalledTimes(records.length) + }) + + it('starts no more after the first failed recovery, and still opens every chat', async () => { + const resolveRecovery = vi.fn(slowFailure) + + await restore({ reconcile: async () => true, resolveRecovery }) + + // Only those already started when the first failed: at most one per chat open at once. + expect(resolveRecovery.mock.calls.length).toBeLessThanOrEqual(4) + expect(restoreRead).toHaveBeenCalledTimes(records.length) + }) + }) + it('does not settle again when a second restore finds the session already open', async () => { restoreRead.mockResolvedValue({ session: { journal: {}, params: {}, child: null }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index facc67a86a92..29ea17845d8a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -72,15 +72,22 @@ async function restoreOneStructuredAgentSessionReadUnderSerialize( export async function restoreStructuredAgentSessionsOnRestart( input: StructuredAgentSessionReadRestoreDeps & { records: AgentSessionRecord[] } ): Promise { - // Once bookkeeping fails in this pass, trying again per chat only waits on the same store again. - let bookkeepingFailed = false + const [first] = input.records + if (!first) { + return + } + // One check for the pass. Each chat checks again while it holds, since another writer can mark + // leases unreconciled mid-pass; after the first failure, retrying per chat only waits on the + // same store again, and the next attach or send settles those chats instead. + let settled = await input.reconcile(first.sessionId) const settleLeases = async (sessionId: string): Promise => { - if (bookkeepingFailed) { - return - } // A session latched in recovery exits here at startup, without waiting for a client. - const settled = (await input.reconcile(sessionId)) && (await input.resolveRecovery(sessionId)) - bookkeepingFailed ||= !settled + if ( + settled && + !((await input.reconcile(sessionId)) && (await input.resolveRecovery(sessionId))) + ) { + settled = false + } } await mapWithConcurrency(input.records, JOURNAL_RESTORE_CONCURRENCY, async ({ sessionId }) => { // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index 77797a5f531b..dcf55f3207c2 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -281,25 +281,30 @@ describe('restoring the chat tabs open at quit', () => { } }) - it('costs two lock waits in all while the lock stays held, however many chats are open', async () => { - const chats = Array.from({ length: 8 }, (_, index) => `chat-${index}-000${index}`) - const records = chats.map((sessionId) => chatRecord(sessionId)) - await seedStore(records, { visible: chats }) - await seedHistory(records) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const { runtime, published } = startupRuntime({ - afterInstall: () => { - lock.failing = true - } - }) + // Each refused take stands for one lock wait of about 3 s. + it.each([1, 4, 8])( + 'waits on a held lock once per startup step with %i chats open', + async (count) => { + const chats = Array.from({ length: count }, (_, index) => `chat-${index}-000${index}`) + const records = chats.map((sessionId) => chatRecord(sessionId)) + await seedStore(records, { visible: chats }) + await seedHistory(records) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { runtime, published } = startupRuntime({ + afterInstall: () => { + lock.failing = true + } + }) - await runtime.prepareStructuredAgentSessionStartupRestoration() - await runtime.restoreStructuredAgentSessionTabs() + await runtime.prepareStructuredAgentSessionStartupRestoration() + const prepared = lock.refused + await runtime.restoreStructuredAgentSessionTabs() - expect(published()).toHaveLength(8) - // One for the startup reconcile, one for the restore's; the restore writes no tab. - expect(lock.refused).toBe(2) - }) + expect(published()).toHaveLength(count) + expect(prepared).toBe(1) + expect(lock.refused - prepared).toBe(1) + } + ) describe('on a legacy profile, with no tab index yet', () => { const legacyChats = () => [ @@ -328,6 +333,28 @@ describe('restoring the chat tabs open at quit', () => { expect(writes.visibility).not.toHaveBeenCalled() }) + it('waits on a held lock once more, for that one write', async () => { + const records = legacyChats() + await seedStore(records) + await seedHistory(records) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { runtime, published } = startupRuntime({ + profileChats: [CHAT_A, CHAT_B], + afterInstall: () => { + lock.failing = true + } + }) + + await runtime.prepareStructuredAgentSessionStartupRestoration() + const prepared = lock.refused + await runtime.restoreStructuredAgentSessionTabs() + + expect(published()).toHaveLength(2) + expect(prepared).toBe(1) + // The restore's lease check and the seed. + expect(lock.refused - prepared).toBe(2) + }) + it('still lists the chats when that write fails, and leaves the index to seed again', async () => { const records = legacyChats() const { path } = await seedStore(records) From a77e8245d166bc319251d79d5c04d5044377e4f8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:27:42 -0700 Subject: [PATCH 006/107] docs(native-chat): correct restore comments and a test name to match the final design --- .../structured-agent-session-restart-reconcile.ts | 5 +++-- ...tructured-agent-session-startup-reconcile-failure.test.ts | 4 ++-- src/main/runtime/agent-session-tab-table.ts | 1 - ...rca-runtime-restore-structured-agent-session-tabs-once.ts | 2 +- .../structured-agent-session-startup-tab-restore.test.ts | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index be2d0b6c1513..23f0e0c6bfae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -45,8 +45,9 @@ export function createRestartReconciler(deps: { } } -/** Reports each distinct failure once, until `clear` says the bookkeeping settled again. Every read - * re-runs a failing reconcile, so without this a failing store would log on every chat list. */ +/** Reports each distinct failure once, until `clear` says the bookkeeping settled again: the + * startup check, the restore pass and a restore retried after a failed journal open would each + * log the same store failure. */ export type ReaderBookkeepingFailures = { report: (failure: unknown) => void clear: () => void diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts index 8033843cd41e..ed18328663c5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts @@ -90,7 +90,7 @@ async function relaunch( mintSpawnToken: () => 'spawn-next', probeOwner, stopOwnerProcess: () => { - throw new Error('a read must not stop an owner') + throw new Error('an owner not proven alive must not be stopped') }, now: () => NOW, onLeaseReconcileFailure @@ -180,7 +180,7 @@ it('restores a chat for reading from a store a newer Orca wrote', async () => { expect(await readFile(path)).toEqual(bytes) }) -// A chat whose owner could not be proven gone is left recovering; resolving that is the writer's. +// A chat whose owner could not be proven gone is left recovering; the next attach or send retries it. it('restores a chat for reading when resolving its recovery cannot write the store', async () => { const { host, store, onLeaseReconcileFailure } = await relaunch(undefined, async () => ({ outcome: 'indeterminate', diff --git a/src/main/runtime/agent-session-tab-table.ts b/src/main/runtime/agent-session-tab-table.ts index 840356cf0cf0..72551d563cab 100644 --- a/src/main/runtime/agent-session-tab-table.ts +++ b/src/main/runtime/agent-session-tab-table.ts @@ -131,7 +131,6 @@ export function setAgentSessionTabVisibility( } } -/** Shows each session that still has a record. */ export function showAgentSessionTabs( state: AgentSessionStoreState, sessionIds: readonly string[] diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 1eea896b5dce..10dd920b253b 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -84,7 +84,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu targets, restored.map((session) => session.sessionId) ) - // The store already lists every tab projected here, so the restore writes nothing else. + // Past the seed, projecting records nothing. for (const replacement of host?.conversationReplacements?.() ?? []) { this.replaceStructuredAgentSessionTab(replacement) } diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index dcf55f3207c2..61fa020e4276 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -355,7 +355,7 @@ describe('restoring the chat tabs open at quit', () => { expect(lock.refused - prepared).toBe(2) }) - it('still lists the chats when that write fails, and leaves the index to seed again', async () => { + it('still lists the chats when that write fails, and leaves the index absent', async () => { const records = legacyChats() const { path } = await seedStore(records) await seedHistory(records) From 2a030d0c280663a7cce5561f2bb1140f027f56d3 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:49:46 -0700 Subject: [PATCH 007/107] fix(native-chat): keep the post-list chat restore to the chats still listed, one failure per chat The restore that reads each chat's history now runs after the tab list, while the app is already usable, so three things it did before the list existed now show: - A chat whose history failed to open ended one of the restore's four lanes. With four such chats, every chat after them got no status row until opened. Each chat now settles on its own and each failure is logged once. - A chat closed before the restore reached it was opened again and its status row republished. The restore now skips, under the chat's own serialized step, a chat whose tab is no longer listed. - After quit began, the restore kept opening chats. It now stops. Tests: the startup listing tests share the existing rest test rig, extended to several chats, instead of a parallel rig; the status parity test no longer leaves a temp directory behind. Comments that said an unreconciled lease admits no writer, or that a write waits for its own chat's check, now say what happens. --- ...red-agent-session-conversation-lifetime.ts | 9 +- .../structured-agent-session-host-tabs.ts | 9 + .../structured-agent-session-host.ts | 3 +- ...gent-session-persisted-tab-listing.test.ts | 47 ++-- ...ed-agent-session-readable-restorer.test.ts | 2 + .../structured-agent-session-rest-test-rig.ts | 186 +++++++++++--- ...uctured-agent-session-restart-reconcile.ts | 11 +- ...ured-agent-session-restart-restore.test.ts | 10 + ...tructured-agent-session-restart-restore.ts | 39 ++- ...gent-session-restore-after-listing.test.ts | 82 ++++++ ...ent-session-restore-without-import.test.ts | 2 + ...ent-session-restored-status-parity.test.ts | 33 +-- .../structured-agent-session-reveal.ts | 4 +- ...-agent-session-startup-listing-test-rig.ts | 242 ------------------ ...red-agent-session-write-lease-wait.test.ts | 30 ++- ...structured-agent-session-create-support.ts | 4 +- ...-agent-session-startup-tab-restore.test.ts | 8 +- ...-agent-session-tab-listing-startup.test.ts | 71 ++--- 18 files changed, 408 insertions(+), 384 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index f69129183b4a..44a427bd1cc9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -20,6 +20,7 @@ import { import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { StructuredAgentSessionIdleSweep } from './structured-agent-session-idle-sweep' import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' +import { sessionTabListed } from './structured-agent-session-host-tabs' import { adapterSupportsRecord } from './structured-agent-session-provider-support' export type StructuredAgentSessionConversationLifetime = ReturnType< @@ -54,11 +55,8 @@ export function createStructuredAgentSessionConversationLifetime(host: { closeStructuredAgentSessionConversationUnderSerialize( { sessions, - closeStatus: (id) => { - const tabs = deps().store.getVisibleSessionTabIndex() - // A legacy store cannot say, so the row stays; restart is the boundary that forgets. - host.closeStatus(id, { listed: !tabs.present || tabs.sessionIds.includes(id) }) - } + // A legacy store cannot say, so the row stays; restart is the boundary that forgets. + closeStatus: (id) => host.closeStatus(id, { listed: sessionTabListed(deps().store, id) }) }, sessionId ) @@ -89,6 +87,7 @@ export function createStructuredAgentSessionConversationLifetime(host: { return { idleSweep, stopAgent, + isDisposed: (): boolean => disposed, /** Quit has begun: nothing opens a conversation or sweeps one after this. */ dispose: (): void => { disposed = true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts index ee13814b485b..0facc7cb037e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts @@ -32,6 +32,15 @@ export function setStructuredAgentSessionTabVisibility( return host.deps.store.setSessionTabVisibility(sessionId, visible, tabId) } +/** Whether the chat still has its tab. A legacy store with no tab index cannot say, so yes. */ +export function sessionTabListed( + store: { getVisibleSessionTabIndex: () => { present: boolean; sessionIds: string[] } }, + sessionId: string +): boolean { + const tabs = store.getVisibleSessionTabIndex() + return !tabs.present || tabs.sessionIds.includes(sessionId) +} + export type StructuredAgentSessionTab = { sessionId: string workspaceId: string diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 24f1de51ab2d..f82b8a2cec15 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -142,6 +142,7 @@ export class StructuredAgentSessionHost { resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), hasSession: this.hasSession, + isDisposed: () => this.lifetime.isDisposed(), // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. onReadable: this.conversationDelivery.adoptOpened @@ -275,7 +276,7 @@ export class StructuredAgentSessionHost { publish: (sessionId, journal) => this.subscribers.publish(sessionId, journal), flushStreamedEvents: this.flushStreamedEvents, conversation: this.lifetime.conversation, - // Every write funnels through here, so none acts on a lease the startup check has not reached. + // Every write funnels through here: one to a chat with an unchecked lease waits for the check. serialize: serializeAfterLeaseCheck(this.deps.store, this.reconcileLeases, this.serialize), openConversation: this.conversationDelivery.open, ensureAgent: (sessionId) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts index f14a53ee673a..b75ab410111e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts @@ -6,14 +6,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { isAgentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' import { - createStartupRig, - type StartupRig -} from './structured-agent-session-startup-listing-test-rig' + createRestTestRig, + restTestChat, + restTestOpens, + sendRestTestMessage, + type RestTestRig +} from './structured-agent-session-rest-test-rig' -let rig: StartupRig +let rig: RestTestRig beforeEach(async () => { - rig = await createStartupRig() + rig = await createRestTestRig() }) afterEach(async () => { @@ -30,8 +33,8 @@ const tab = (sessionId: string, workspaceId = 'workspace-1') => ({ describe('the tab list after a restart', () => { it('lists every chat it is given without opening one', async () => { - await rig.chat('session-a', { message: 'one' }) - await rig.chat('session-b', { message: 'two' }) + await restTestChat(rig, 'session-a', { message: 'one' }) + await restTestChat(rig, 'session-b', { message: 'two' }) await rig.crash() const host = await rig.boot() @@ -39,16 +42,16 @@ describe('the tab list after a restart', () => { tab('session-a'), tab('session-b') ]) - expect(rig.historyFilePath).not.toHaveBeenCalled() + expect(rig.adapter.historyFilePath).not.toHaveBeenCalled() }) it('keeps a chat with no history on disk, which then reads empty and takes a send (T3)', async () => { // Its first start failed before the conversation opened, so no history was ever written. - rig.acquire.mockRejectedValueOnce(new Error('spawn failed')) + rig.adapter.acquire.mockRejectedValueOnce(new Error('spawn failed')) vi.spyOn(console, 'warn').mockImplementation(() => undefined) - await expect(rig.chat('session-new')).rejects.toThrow(/attach refused/) + await expect(restTestChat(rig, 'session-new')).rejects.toThrow(/attach refused/) await rig.store.setSessionTabVisibility('session-new', true) - await rig.chat('session-old', { message: 'hello' }) + await restTestChat(rig, 'session-old', { message: 'hello' }) await rig.crash() const host = await rig.boot() const ids = ['session-new', 'session-old'] @@ -57,22 +60,22 @@ describe('the tab list after a restart', () => { expect(host.listSessionTabs(ids)).toEqual([tab('session-new'), tab('session-old')]) // The restore founds no history for it; the tab stands anyway. - expect(rig.opensOf('session-new')).toBe(0) + expect(restTestOpens(rig, 'session-new')).toBe(0) const page = await host.history({ sessionId: 'session-new', direction: 'tail' }) expect(page.ok && page.page.items).toEqual([]) - const sent = await rig.send('session-new', 'first words') + const sent = await sendRestTestMessage(rig, 'session-new', 'first words') expect(sent).toMatchObject({ ok: true }) - await vi.waitFor(() => expect(rig.dispatch).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) const after = await host.history({ sessionId: 'session-new', direction: 'tail' }) expect(JSON.stringify(after.ok && after.page)).toContain('first words') }) it('keeps a chat whose history cannot be opened, and its read says why (T4)', async () => { - await rig.chat('session-bad', { message: 'lost' }) - await rig.chat('session-good', { message: 'kept' }) + await restTestChat(rig, 'session-bad', { message: 'lost' }) + await restTestChat(rig, 'session-good', { message: 'kept' }) await rig.crash() const host = await rig.boot() - rig.historyFilePath.mockImplementation(async (sessionId) => { + rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { if (sessionId === 'session-bad') { throw new Error('EACCES: permission denied') } @@ -94,9 +97,9 @@ describe('the tab list after a restart', () => { }) it('lists in the order it is given, once each, whatever order the records are in (T7)', async () => { - await rig.chat('session-c') - await rig.chat('session-a') - await rig.chat('session-b') + await restTestChat(rig, 'session-c') + await restTestChat(rig, 'session-a') + await restTestChat(rig, 'session-b') await rig.crash() const host = await rig.boot() @@ -108,8 +111,8 @@ describe('the tab list after a restart', () => { }) it('lists no chat without a record, and none this host cannot serve (T10, regression guard)', async () => { - await rig.chat('session-served', { message: 'here' }) - await rig.chat('session-gated', { workspaceId: 'workspace-gated', message: 'there' }) + await restTestChat(rig, 'session-served', { message: 'here' }) + await restTestChat(rig, 'session-gated', { workspaceId: 'workspace-gated', message: 'there' }) await rig.crash() const host = await rig.boot() rig.unsupportedWorkspaceIds.add('workspace-gated') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index 09cab618ab9a..12e52293801f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -43,6 +43,8 @@ describe('StructuredAgentSessionReadableRestorer', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts index 782c1f161156..6ae31f023cf0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts @@ -1,6 +1,8 @@ // A real host over a real store and journal, with a scripted provider and a clock the test moves, -// for the tests of a conversation that outlives its agent. The idle sweep runs on its own short -// interval; a test moves `clock.now` past the idle window and waits for the outcome. +// for the tests of a conversation that outlives its agent and of what a restarted host owes. The +// idle sweep runs on its own short interval; a test moves `clock.now` past the idle window and +// waits for the outcome. Every journal open calls `historyFilePath` once, so that mock is the open +// counter, and holding it holds the open. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -8,6 +10,8 @@ import { join } from 'node:path' import { expect, vi, type Mock } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionMutationEnvelope, AgentSessionStatusEvent, @@ -21,6 +25,7 @@ import type { import { StructuredAgentSessionHost } from './structured-agent-session-host' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { + HOST_TEST_LOCATION, HOST_TEST_NOW, HOST_TEST_SESSION as SESSION, HOST_TEST_THREAD as THREAD, @@ -45,6 +50,8 @@ export type RestTestAdapter = { > backgroundTaskState: Mock> readOptions: Mock> + /** Called once per journal open, with the session id; replace its implementation to hold one. */ + historyFilePath: Mock<(sessionId: string) => Promise> } export type RestTestRig = { @@ -55,36 +62,55 @@ export type RestTestRig = { clock: { now: number } statusEvents: AgentSessionStatusEvent[] sink: { publish: Mock; forget: Mock } + probeOwner: Mock<(record: AgentSessionRecord) => Promise> + /** Workspaces this host's adapter does not serve, as a platform gate would. */ + unsupportedWorkspaceIds: Set /** Opens a fresh host over the same store and journals: what a restart leaves behind. */ restart: (deps?: Partial) => Promise + /** A crash: nothing settles or flushes; the process's timers and handles simply stop. */ + crash: () => Promise + /** A new app run over the same files, with no flush first; the mocks and status stream start + * empty, so they hold what the new run saw and nothing the previous one did. */ + boot: (deps?: Partial) => Promise dispose: () => Promise } let ordinal = 0 let generations = 0 -export function acceptedDispatch(): AgentSessionDispatchOutcome { +/** The rig's own chat keeps the shared test thread; any other chat gets a thread of its own. */ +function restTestThread(sessionId: string): string { + return sessionId === SESSION ? THREAD : `thread-${sessionId}` +} + +export function acceptedDispatch(sessionId = SESSION): AgentSessionDispatchOutcome { ordinal += 1 return { state: 'accepted', - providerIdentity: { provider: 'codex', threadId: THREAD, turnId: `turn-${ordinal}`, ordinal } + providerIdentity: { + provider: 'codex', + threadId: restTestThread(sessionId), + turnId: `turn-${ordinal}`, + ordinal + } } } export function restTestSend( text: string, - fence = 1 + fence: number | null = 1, + sessionId = SESSION ): { envelope: AgentSessionMutationEnvelope; body: AgentJournalMessageItem } { const body = hostTestMessage(text) return { body, envelope: { - sessionId: SESSION, + sessionId, clientOperationId: hostTestOperationId(), expectedRuntimeFence: fence, payloadFingerprint: computeAgentSessionPayloadFingerprint({ method: 'agentSession.send', - sessionId: SESSION, + sessionId, fields: { body } }) } @@ -117,27 +143,31 @@ export function collectSubscriber(): { return { events, emit: (event) => events.push(event) } } +/** `root` is handed to the rig, which removes it on dispose; by default it makes its own. */ export async function createRestTestRig( - deps: Partial = {} + deps: Partial = {}, + options: { root?: string } = {} ): Promise { resetHostTestOperationIds() ordinal = 0 - const root = await mkdtemp(join(tmpdir(), 'orca-rest-')) + const root = options.root ?? (await mkdtemp(join(tmpdir(), 'orca-rest-'))) const clock = { now: HOST_TEST_NOW } const statusEvents: AgentSessionStatusEvent[] = [] const sink = { publish: vi.fn(), forget: vi.fn() } - let store = await AgentSessionRecordStore.open({ - directory: join(root, 'store'), - hostId: 'local' - }) + const probeOwner: RestTestRig['probeOwner'] = vi.fn(async () => ({ outcome: 'pid-absent' })) + const unsupportedWorkspaceIds = new Set() + const openStore = () => + AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + let store = await openStore() const adapter: RestTestAdapter = { - acquire: vi.fn(async ({ fence, spawnToken }) => ({ + acquire: vi.fn(async ({ fence, spawnToken, identity }) => ({ acquisitionGeneration: `generation-${++generations}`, process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, link: { - linkId: `link-${fence}`, - handle: { provider: 'codex' as const, threadId: THREAD }, - origin: store.getRecord(SESSION)?.providerHandleChain.length + linkId: + identity.sessionId === SESSION ? `link-${fence}` : `link-${identity.sessionId}-${fence}`, + handle: { provider: 'codex' as const, threadId: restTestThread(identity.sessionId) }, + origin: store.getRecord(identity.sessionId)?.providerHandleChain.length ? ('resumed' as const) : ('created' as const), mintedAtFence: fence, @@ -145,16 +175,20 @@ export async function createRestTestRig( } })), closeSession: vi.fn(async () => true), - dispatch: vi.fn(async () => acceptedDispatch()), + dispatch: vi.fn(async (input) => acceptedDispatch(input.sessionId)), acknowledgeSessionRelease: vi.fn(), backgroundTaskState: vi.fn(() => undefined), - readOptions: vi.fn(async () => ({ models: [], current: { model: 'gpt-live' } })) + readOptions: vi.fn(async () => ({ models: [], current: { model: 'gpt-live' } })), + historyFilePath: vi.fn(async (_sessionId: string): Promise => null) } - const hostFor = (overrides: Partial) => - new StructuredAgentSessionHost({ + const hostFor = (overrides: Partial) => { + const host = new StructuredAgentSessionHost({ store, adapter: { ...adapter, + historyFilePath: ({ identity }) => adapter.historyFilePath(identity.sessionId), + supportsCreate: (location, agent) => + agent === 'codex' && !unsupportedWorkspaceIds.has(location.workspaceId), releaseAcquisition: vi.fn(async () => true), cancelTurn: async () => ({ cancelled: true }), answerPrompt: async ({ commit }) => commit(), @@ -163,13 +197,22 @@ export async function createRestTestRig( journalDatabase: openTestJournalHostDatabase(root), claimKeyId: 'key-1', mintSpawnToken: () => 'spawn-a', - probeOwner: async () => ({ outcome: 'pid-absent' }), + probeOwner, now: () => clock.now, statusSink: sink, idleSweep: { intervalMs: SWEEP_INTERVAL_MS }, ...deps, ...overrides }) + host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) + return host + } + const reopen = async (overrides: Partial) => { + store = await openStore() + rig.store = store + rig.host = hostFor(overrides) + return rig.host + } const rig: RestTestRig = { root, store, @@ -178,38 +221,107 @@ export async function createRestTestRig( clock, statusEvents, sink, + probeOwner, + unsupportedWorkspaceIds, restart: async (overrides = {}) => { await rig.host.flushAllStreamedEvents().catch(() => undefined) - store = await AgentSessionRecordStore.open({ - directory: join(root, 'store'), - hostId: 'local' - }) - rig.store = store - rig.host = hostFor(overrides) - rig.host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) - return rig.host + return reopen(overrides) + }, + crash: async () => { + const { runtimeState, lifetime, sessions } = rig.host.collaboratorsForTests() + await runtimeState.stopLeaseRenewal() + lifetime.dispose() + for (const session of sessions.values()) { + await session.journal.close() + } + sessions.clear() + }, + boot: (overrides = {}) => { + for (const mock of [adapter.historyFilePath, adapter.acquire, adapter.dispatch, probeOwner]) { + mock.mockClear() + } + sink.publish.mockClear() + sink.forget.mockClear() + statusEvents.length = 0 + return reopen(overrides) }, dispose: async () => { await rig.host.flushAllStreamedEvents().catch(() => undefined) await rm(root, { recursive: true, force: true }) } } - rig.host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) return rig } -/** Creates the chat, lists its tab, and sends one message so its journal is on disk. */ -export async function foundRestTestChat(rig: RestTestRig): Promise { - const attached = await rig.host.attach(REST_TEST_CALLER, hostTestAttachParams(null)) +/** Creates the chat, lists its tab unless told not to, and sends `message` when one is given, + * waiting for its dispatch. */ +export async function restTestChat( + rig: RestTestRig, + sessionId: string, + options: { workspaceId?: string; listed?: boolean; message?: string } = {} +): Promise { + const fence = rig.store.getRecord(sessionId)?.lease.runtimeFence ?? null + const attached = await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(fence, { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: fence, + payloadFingerprint: '' + }, + location: { ...HOST_TEST_LOCATION, workspaceId: options.workspaceId ?? 'workspace-1' }, + providerHandle: { kind: 'codex', threadId: restTestThread(sessionId) } + }) + ) if (!attached.ok) { throw new Error(`attach refused: ${attached.refusal.code}`) } - await rig.store.setSessionTabVisibility(SESSION, true) - const sent = await rig.host.send(REST_TEST_CALLER, restTestSend('hello', attached.fence)) + if (options.listed !== false) { + await rig.store.setSessionTabVisibility(sessionId, true) + } + if (options.message === undefined) { + return + } + const dispatched = rig.adapter.dispatch.mock.calls.length + const sent = await rig.host.send( + REST_TEST_CALLER, + restTestSend(options.message, attached.fence, sessionId) + ) if (!sent.ok) { throw new Error(`send refused: ${sent.refusal.code}`) } - await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalled()) + await vi.waitFor(() => expect(rig.adapter.dispatch.mock.calls.length).toBeGreaterThan(dispatched)) +} + +/** Creates the rig's own chat, lists its tab, and sends one message so its journal is on disk. */ +export function foundRestTestChat(rig: RestTestRig): Promise { + return restTestChat(rig, SESSION, { message: 'hello' }) +} + +/** A send from a client that has not attached this run, so it names no fence. */ +export function sendRestTestMessage(rig: RestTestRig, sessionId: string, text: string) { + return rig.host.send(REST_TEST_CALLER, restTestSend(text, null, sessionId)) +} + +export function restTestOpens(rig: RestTestRig, sessionId: string): number { + return rig.adapter.historyFilePath.mock.calls.filter(([id]) => id === sessionId).length +} + +/** The newest row the status stream carried for a session. */ +export function latestRestTestStatus(rig: RestTestRig, sessionId: string) { + for (const event of rig.statusEvents.toReversed()) { + if (event.type === 'status' && event.session.sessionId === sessionId) { + return event.session + } + if (event.type === 'snapshot') { + const found = event.sessions.find((session) => session.sessionId === sessionId) + if (found) { + return found + } + } + } + return undefined } /** Runs one sweep pass now, for a test that set `idleSweep.intervalMs` out of reach. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index c4a6df4be312..1c4274da9719 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -106,11 +106,12 @@ function failureKey(failure: unknown): string { return String(failure) } -/** The chat's queue, entered by a write only once that chat's lease is checked. Startup does not - * wait for the lease check, so a write can reach a chat whose lease is still the previous run's; - * it waits for that check, or runs it. With none owed it enqueues at once, keeping its place. A - * check that fails gates nothing: the lease stays unreconciled, which admits no writer, and the - * next start checks again. */ +/** The chat's queue, entered by a write to a chat whose lease is still the previous run's only + * after the lease check, which startup does not wait for. The write waits for that check (one pass + * over every unreconciled lease, not only this chat's), or runs it. With none owed it enqueues at + * once, keeping its place. A check that fails gates nothing: a conversation write (a send, a Stop) + * is still admitted, a provider-child write is refused while the lease stays unreconciled, and an + * agent start reconciles again and refuses. */ export function serializeAfterLeaseCheck( store: Pick, reconcile: (sessionId: string) => Promise, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 0cffa1e32c75..7540e8159744 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -66,6 +66,8 @@ describe('restart journal restoration', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) @@ -107,6 +109,8 @@ describe('restart journal restoration', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) ticking = false @@ -157,6 +161,8 @@ describe('restart journal restoration', () => { }, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: (_sessionId, readable) => { calls.push(readable === restored ? 'onReadable:restored' : 'onReadable') } @@ -184,6 +190,8 @@ describe('restart journal restoration', () => { ...bookkeeping, serialize: async (_sessionId, task) => task(), hasSession: () => false, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) @@ -231,6 +239,8 @@ describe('restart journal restoration', () => { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: () => true, + isDisposed: () => false, + isListed: () => true, onReadable: () => undefined }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 29ea17845d8a..fd621db47cbe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -13,7 +13,7 @@ import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { mapWithConcurrency } from '../../../shared/map-with-concurrency' +import { mapSettledWithConcurrency } from '../../../shared/map-with-concurrency' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { OpenedStructuredAgentSessionConversation, @@ -34,6 +34,10 @@ export type StructuredAgentSessionReadRestoreDeps = { resolveRecovery: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean + /** Quit has begun: the restore opens nothing more. */ + isDisposed: () => boolean + /** Whether the chat still has its tab: one closed while the restore ran stays closed. */ + isListed: (sessionId: string) => boolean onReadable: ( sessionId: string, opened: OpenedStructuredAgentSessionConversation @@ -46,6 +50,9 @@ async function restoreOneStructuredAgentSessionRead( sessionId: string, settleLeases: (sessionId: string) => Promise ): Promise { + if (input.isDisposed()) { + return + } await settleLeases(sessionId) await input.serialize(sessionId, () => restoreOneStructuredAgentSessionReadUnderSerialize(input, sessionId) @@ -54,9 +61,16 @@ async function restoreOneStructuredAgentSessionRead( /** The serialized half of the restore. */ async function restoreOneStructuredAgentSessionReadUnderSerialize( - input: Pick, + input: Pick< + StructuredAgentSessionReadRestoreDeps, + 'openDeps' | 'hasSession' | 'isDisposed' | 'isListed' | 'onReadable' + >, sessionId: string ): Promise { + // Checked under the chat's serialize, where its close and quit's teardown also run. + if (input.isDisposed() || !input.isListed(sessionId)) { + return + } if (input.hasSession(sessionId)) { // A read or a send mid-restore already opened this one. return @@ -89,9 +103,22 @@ export async function restoreStructuredAgentSessionsOnRestart( settled = false } } - await mapWithConcurrency(input.records, JOURNAL_RESTORE_CONCURRENCY, async ({ sessionId }) => { - // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. - await yieldToEventLoop() - await restoreOneStructuredAgentSessionRead(input, sessionId, settleLeases) + const results = await mapSettledWithConcurrency( + input.records, + JOURNAL_RESTORE_CONCURRENCY, + async ({ sessionId }) => { + // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. + await yieldToEventLoop() + await restoreOneStructuredAgentSessionRead(input, sessionId, settleLeases) + } + ) + // One chat's failure costs only that chat, which opens again when it is read. + results.forEach((result, index) => { + if (result.status === 'rejected') { + console.warn('[structured-agent-session] restoring a chat for reading failed', { + sessionId: input.records[index]?.sessionId, + error: result.reason + }) + } }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts new file mode 100644 index 000000000000..9752089002b9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts @@ -0,0 +1,82 @@ +// The restore pass runs after the tab list is out, so the app is already usable while it works +// through the chats. A chat closed, or a host quit, before the pass reaches it stays that way. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { + createRestTestRig, + restTestChat, + restTestOpens, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const IDS = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] +const LAST = 'session-5' + +let rig: RestTestRig +let releaseHeld = (): void => undefined + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + releaseHeld() + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +/** Boots over a crashed run and starts the pass with the first four opens held, so it has not + * reached the last chat. */ +async function bootWithHeldPass() { + for (const sessionId of IDS) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.crash() + const host = await rig.boot() + const held = Promise.withResolvers() + releaseHeld = held.resolve + rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { + if (sessionId !== LAST) { + await held.promise + } + return null + }) + const pass = host.restoreReadableSessions(IDS) + await vi.waitFor(() => expect(rig.adapter.historyFilePath).toHaveBeenCalledTimes(4)) + return { host, pass, release: held.resolve } +} + +const publishedFor = (sessionId: string): number => + rig.sink.publish.mock.calls.filter(([summary]) => summary.sessionId === sessionId).length + +describe('the restore pass after the tab list', () => { + it('does not reopen a chat closed before the pass reached it', async () => { + const { host, pass, release } = await bootWithHeldPass() + + // What closing the tab does. + await host.setSessionTabVisibility(LAST, false) + await host.close(LAST) + const publishedAtClose = publishedFor(LAST) + release() + await pass + + expect(host.hasSession(LAST)).toBe(false) + expect(restTestOpens(rig, LAST)).toBe(0) + expect(publishedFor(LAST)).toBe(publishedAtClose) + expect(host.hasSession('session-1')).toBe(true) + }) + + it('opens nothing more once the host is quitting', async () => { + const { host, pass, release } = await bootWithHeldPass() + + const teardown = host.flushAllStreamedEvents() + release() + await teardown + await pass + + expect(restTestOpens(rig, LAST)).toBe(0) + expect(host.hasSession(LAST)).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts index 3290ba05f0de..7a5d2402d8e2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts @@ -236,6 +236,8 @@ async function restore(sessionIds: readonly string[]) { resolveRecovery: async () => true, serialize: async (_sessionId, task) => task(), hasSession: (sessionId) => sessions.has(sessionId), + isDisposed: () => false, + isListed: () => true, onReadable: (sessionId, opened) => { sessions.set(sessionId, opened.session) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts index b0c033c2f84b..ac399f72e8d8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts @@ -7,12 +7,13 @@ import { afterEach, expect, it, vi } from 'vitest' import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' import { - createStartupRig, - latestStatus, - type StartupRig -} from './structured-agent-session-startup-listing-test-rig' + createRestTestRig, + latestRestTestStatus, + restTestChat, + type RestTestRig +} from './structured-agent-session-rest-test-rig' -const rigs: StartupRig[] = [] +const rigs: RestTestRig[] = [] afterEach(async () => { for (const rig of rigs.splice(0)) { @@ -26,14 +27,14 @@ const CORPUS = ['session-settled', 'session-quiet', 'session-crashed', 'session- /** The chats a restart finds: two a clean quit settled, one cut off mid-turn by a crash, and one * whose send the provider only admitted, left pending below a fence that has since moved. */ -async function seedCorpus(rig: StartupRig): Promise { - await rig.chat('session-settled', { message: 'finished work' }) - await rig.chat('session-quiet') +async function seedCorpus(rig: RestTestRig): Promise { + await restTestChat(rig, 'session-settled', { message: 'finished work' }) + await restTestChat(rig, 'session-quiet') await rig.host.flushAllStreamedEvents() await rig.boot() - await rig.chat('session-crashed', { message: 'cut off' }) - rig.dispatch.mockResolvedValueOnce({ state: 'admitted' }) - await rig.chat('session-pending', { message: 'only admitted' }) + await restTestChat(rig, 'session-crashed', { message: 'cut off' }) + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, 'session-pending', { message: 'only admitted' }) await rig.crash() await rig.store.transitionHandoff('session-pending', (record) => ({ ...record, @@ -42,14 +43,14 @@ async function seedCorpus(rig: StartupRig): Promise { } it('ends every chat where a direct read of it lands, and shows no pre-crash work (T8)', async () => { - const passRig = await createStartupRig() + const passRig = await createRestTestRig() rigs.push(passRig) await seedCorpus(passRig) // One copy of the files for each boot, taken while nothing is writing. closeTestJournalHostDatabases() const readRoot = `${passRig.root}-read` await cp(passRig.root, readRoot, { recursive: true }) - const readRig = await createStartupRig(readRoot) + const readRig = await createRestTestRig({}, { root: readRoot }) rigs.push(readRig) // Startup's shape: the lease check started, the list answered, then the pass. @@ -67,10 +68,10 @@ it('ends every chat where a direct read of it lands, and shows no pre-crash work await readCheck // `updatedAt` is when the row was projected, a wall-clock read that differs between two boots. - const rows = (rig: StartupRig): Record> => + const rows = (rig: RestTestRig): Record> => Object.fromEntries( CORPUS.flatMap((sessionId) => { - const summary = latestStatus(rig, sessionId) + const summary = latestRestTestStatus(rig, sessionId) if (!summary) { return [] } @@ -80,7 +81,7 @@ it('ends every chat where a direct read of it lands, and shows no pre-crash work ) await vi.waitFor(() => expect(rows(passRig)).toEqual(rows(readRig))) for (const sessionId of CORPUS) { - expect(latestStatus(passRig, sessionId)).toBeDefined() + expect(latestRestTestStatus(passRig, sessionId)).toBeDefined() } const crashedRows = passRig.statusEvents.flatMap((event) => event.type === 'status' && event.session.sessionId === 'session-crashed' ? [event.session] : [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 262c9e18d3c6..ad03ad82e8cd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -12,6 +12,7 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' +import { sessionTabListed } from './structured-agent-session-host-tabs' import { adapterSupportsRecord } from './structured-agent-session-provider-support' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' @@ -62,7 +63,7 @@ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], - 'openDeps' | 'supportsRecord' | 'reconcile' | 'resolveRecovery' + 'openDeps' | 'supportsRecord' | 'reconcile' | 'resolveRecovery' | 'isListed' > & { reconcileLeases: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise @@ -77,6 +78,7 @@ export function createStructuredAgentSessionHostRestore( const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: deps, supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), + isListed: (sessionId) => sessionTabListed(deps.store, sessionId), reconcile, // The next attach or send resolves recovery again, strictly, before it acts. resolveRecovery: (sessionId) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts deleted file mode 100644 index 479f9b4df51b..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig.ts +++ /dev/null @@ -1,242 +0,0 @@ -// Real hosts over one real record store and journal database, booted in turn the way app runs -// follow each other, for the tests of what a restarted host lists and owes before anyone opens a -// chat. Every journal open calls the adapter's `historyFilePath` once, so that is the open counter, -// and holding it holds the open. - -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { expect, vi, type Mock } from 'vitest' -import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' -import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { - AgentSessionMutationEnvelope, - AgentSessionStatusEvent -} from '../../../shared/agent-session-wire' -import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' -import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { StructuredAgentSessionHost } from './structured-agent-session-host' -import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' -import { - HOST_TEST_LOCATION, - HOST_TEST_NOW, - hostTestAttachParams, - hostTestMessage, - hostTestOperationId, - resetHostTestOperationIds -} from './structured-agent-session-host-test-data' - -export const STARTUP_CALLER = { callerKey: 'client-1' } - -export type StartupRig = { - root: string - store: AgentSessionRecordStore - host: StructuredAgentSessionHost - acquire: Mock - dispatch: Mock - /** Called once per journal open, with the session id; replace its implementation to hold one. */ - historyFilePath: Mock<(sessionId: string) => Promise> - probeOwner: Mock<(record: AgentSessionRecord) => Promise> - /** Workspaces this host's adapter does not serve, as a platform gate would. */ - unsupportedWorkspaceIds: Set - statusEvents: AgentSessionStatusEvent[] - /** Creates the chat, lists its tab unless told not to, and sends `message` when one is given, - * waiting for its dispatch. */ - chat: ( - sessionId: string, - options?: { workspaceId?: string; listed?: boolean; message?: string } - ) => Promise - send: (sessionId: string, text: string) => ReturnType - opensOf: (sessionId: string) => number - /** A crash: nothing settles or flushes; the process's timers and handles simply stop. */ - crash: () => Promise - /** A new app run over the same files; the counters and status stream start empty. */ - boot: (deps?: Partial) => Promise - dispose: () => Promise -} - -export function startupSendEnvelope( - sessionId: string, - body: ReturnType -): AgentSessionMutationEnvelope { - return { - sessionId, - clientOperationId: hostTestOperationId(), - expectedRuntimeFence: null, - payloadFingerprint: computeAgentSessionPayloadFingerprint({ - method: 'agentSession.send', - sessionId, - fields: { body } - }) - } -} - -function startupAttachParams( - sessionId: string, - workspaceId: string, - fence: number | null -): AgentSessionAttachParams { - return hostTestAttachParams(fence, { - envelope: { - sessionId, - clientOperationId: hostTestOperationId(), - expectedRuntimeFence: fence, - payloadFingerprint: '' - }, - location: { ...HOST_TEST_LOCATION, workspaceId }, - providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } - }) -} - -export async function createStartupRig( - root?: string, - deps: Partial = {} -): Promise { - resetHostTestOperationIds() - const stateRoot = root ?? (await mkdtemp(join(tmpdir(), 'orca-startup-listing-'))) - const directory = join(stateRoot, 'store') - const statusEvents: AgentSessionStatusEvent[] = [] - const historyFilePath = vi.fn(async (_sessionId: string): Promise => null) - const probeOwner: StartupRig['probeOwner'] = vi.fn(async () => ({ outcome: 'pid-absent' })) - const unsupportedWorkspaceIds = new Set() - let ordinal = 0 - const acquire: StartupRig['acquire'] = vi.fn(async ({ fence, spawnToken, identity }) => ({ - process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, - link: { - linkId: `link-${identity.sessionId}-${fence}`, - handle: { provider: 'codex' as const, threadId: `thread-${identity.sessionId}` }, - origin: rig.store.getRecord(identity.sessionId)?.providerHandleChain.length - ? ('resumed' as const) - : ('created' as const), - mintedAtFence: fence, - observedAt: HOST_TEST_NOW - } - })) - const dispatch: StartupRig['dispatch'] = vi.fn(async (input) => { - ordinal += 1 - return { - state: 'accepted', - providerIdentity: { - provider: 'codex', - threadId: `thread-${input.sessionId}`, - turnId: `turn-${ordinal}`, - ordinal - } - } - }) - const open = async (overrides: Partial) => { - const store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) - const host = new StructuredAgentSessionHost({ - store, - adapter: { - acquire, - dispatch, - closeSession: vi.fn(async () => true), - releaseAcquisition: vi.fn(async () => true), - cancelTurn: async () => ({ cancelled: true }), - answerPrompt: async ({ commit }) => commit(), - setOption: async () => undefined, - historyFilePath: ({ identity }) => historyFilePath(identity.sessionId), - supportsCreate: (location, agent) => - agent === 'codex' && !unsupportedWorkspaceIds.has(location.workspaceId) - }, - journalDatabase: openTestJournalHostDatabase(stateRoot), - claimKeyId: 'key-1', - mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, - probeOwner, - now: () => HOST_TEST_NOW, - ...deps, - ...overrides - }) - host.subscribeStatus({ id: 'status', emit: (event) => statusEvents.push(event) }) - return { store, host } - } - const first = await open({}) - const rig: StartupRig = { - root: stateRoot, - ...first, - acquire, - dispatch, - historyFilePath, - probeOwner, - unsupportedWorkspaceIds, - statusEvents, - chat: async (sessionId, options = {}) => { - const attached = await rig.host.attach( - STARTUP_CALLER, - startupAttachParams( - sessionId, - options.workspaceId ?? 'workspace-1', - rig.store.getRecord(sessionId)?.lease.runtimeFence ?? null - ) - ) - if (!attached.ok) { - throw new Error(`attach refused: ${attached.refusal.code}`) - } - if (options.listed !== false) { - await rig.store.setSessionTabVisibility(sessionId, true) - } - if (options.message === undefined) { - return - } - const dispatched = dispatch.mock.calls.length - const sent = await rig.send(sessionId, options.message) - if (!sent.ok) { - throw new Error(`send refused: ${sent.refusal.code}`) - } - await vi.waitFor(() => expect(dispatch.mock.calls.length).toBeGreaterThan(dispatched)) - }, - send: (sessionId, text) => { - const body = hostTestMessage(text) - return rig.host.send(STARTUP_CALLER, { body, envelope: startupSendEnvelope(sessionId, body) }) - }, - opensOf: (sessionId) => historyFilePath.mock.calls.filter(([id]) => id === sessionId).length, - crash: async () => { - const { runtimeState, lifetime, sessions } = rig.host.collaboratorsForTests() - await runtimeState.stopLeaseRenewal() - lifetime.dispose() - for (const session of sessions.values()) { - await session.journal.close() - } - sessions.clear() - }, - boot: async (overrides = {}) => { - // What the new run saw, and nothing the previous one did. - historyFilePath.mockClear() - acquire.mockClear() - dispatch.mockClear() - probeOwner.mockClear() - statusEvents.length = 0 - const next = await open(overrides) - rig.store = next.store - rig.host = next.host - return next.host - }, - dispose: async () => { - await rig.host.flushAllStreamedEvents().catch(() => undefined) - if (!root) { - await rm(stateRoot, { recursive: true, force: true }) - } - } - } - return rig -} - -/** The newest row the status stream carried for a session. */ -export function latestStatus(rig: StartupRig, sessionId: string) { - for (const event of rig.statusEvents.toReversed()) { - if (event.type === 'status' && event.session.sessionId === sessionId) { - return event.session - } - if (event.type === 'snapshot') { - const found = event.sessions.find((session) => session.sessionId === sessionId) - if (found) { - return found - } - } - } - return undefined -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts index 93899e5c17a9..18bb1714d92b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts @@ -1,23 +1,25 @@ // Startup no longer waits for the lease check, so a write can reach a chat whose lease the previous -// app run left unadjudicated. Every write waits for that chat's check (or runs it) before it acts, -// so none is admitted against the old lease and none starts a second agent. +// app run left unadjudicated. Such a write waits for the lease check (or runs it) before it acts, +// so while that check succeeds none is admitted against the old lease. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' import { hostTestOperationId } from './structured-agent-session-host-test-data' import { - createStartupRig, - STARTUP_CALLER, - type StartupRig -} from './structured-agent-session-startup-listing-test-rig' + createRestTestRig, + REST_TEST_CALLER, + restTestChat, + sendRestTestMessage, + type RestTestRig +} from './structured-agent-session-rest-test-rig' -let rig: StartupRig +let rig: RestTestRig // A failed assertion must not leave teardown waiting on a held probe. let releaseProbe = (): void => undefined beforeEach(async () => { - rig = await createStartupRig() + rig = await createRestTestRig() }) afterEach(async () => { @@ -29,7 +31,7 @@ afterEach(async () => { /** Boots over a crashed run and starts the startup lease check with its owner probe held. */ async function bootWithSlowLeaseCheck() { - await rig.chat('session-a', { message: 'before the restart' }) + await restTestChat(rig, 'session-a', { message: 'before the restart' }) await rig.crash() const probe = Promise.withResolvers() const probing = Promise.withResolvers() @@ -64,7 +66,7 @@ describe('a write during a slow startup lease check', () => { const { probe, startupCheck, admittedUnreconciled } = await bootWithSlowLeaseCheck() let answered = false - const sent = rig.send('session-a', 'after the restart').then((result) => { + const sent = sendRestTestMessage(rig, 'session-a', 'after the restart').then((result) => { answered = true return result }) @@ -75,17 +77,17 @@ describe('a write during a slow startup lease check', () => { probe.resolve() await expect(sent).resolves.toMatchObject({ ok: true }) await startupCheck - await vi.waitFor(() => expect(rig.dispatch).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) expect(admittedUnreconciled).toEqual([false]) - expect(rig.acquire).toHaveBeenCalledOnce() + expect(rig.adapter.acquire).toHaveBeenCalledOnce() }) it('holds a Stop until the chat’s check has run', async () => { const { host, probe, startupCheck, admittedUnreconciled } = await bootWithSlowLeaseCheck() const fields = {} - const stopped = host.cancel(STARTUP_CALLER, { + const stopped = host.cancel(REST_TEST_CALLER, { envelope: { sessionId: 'session-a', clientOperationId: hostTestOperationId(), @@ -104,6 +106,6 @@ describe('a write during a slow startup lease check', () => { await expect(stopped).resolves.toMatchObject({ ok: true }) await startupCheck expect(admittedUnreconciled).toEqual([false]) - expect(rig.acquire).not.toHaveBeenCalled() + expect(rig.adapter.acquire).not.toHaveBeenCalled() }) }) diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index 608ddbceebd9..af7c4aa80d67 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -253,8 +253,8 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR } /** Startup's structured step: the host and PTY inventory, awaited; the lease check, started and - * never awaited. Neither the tab list nor paint needs leases, and every write to a chat whose - * lease is unchecked waits for its own check (`serializeAfterLeaseCheck`). */ + * never awaited. Neither the tab list nor paint needs leases, and a write to a chat whose lease + * is unchecked first waits for that check (`serializeAfterLeaseCheck`). */ async prepareStructuredAgentSessionStartupRestoration(): Promise { await this.ensureStructuredAgentSessionInventory() if (this.hasPersistedStructuredAgentSessionStore()) { diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index b5ad33204087..79ad60a5486e 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -1,7 +1,7 @@ -// With native chat on, the renderer's startup also awaits the chat tab restore (`session.tabs.listAll`), -// which reads every chat whose tab was open at quit. That read must not wait on record-store -// bookkeeping: with a saved tab index it writes nothing, and a store that cannot be written costs -// a bounded number of lock waits, not one per chat. +// With native chat on, the renderer's startup also awaits the chat tab list (`session.tabs.listAll`). +// The list opens no chat; a restore it does not wait for then reads every chat whose tab was open at +// quit. Neither may wait on record-store bookkeeping: with a saved tab index they write nothing, and +// a store that cannot be written costs a bounded number of lock waits, not one per chat. import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 239f10627660..9c8687d524a0 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -7,10 +7,11 @@ import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' import { closeTestJournalHostDatabases } from '../native-chat/agent-session-journal/journal-host-database-test-support' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { - createStartupRig, - latestStatus, - type StartupRig -} from '../native-chat/agent-session-wire/structured-agent-session-startup-listing-test-rig' + createRestTestRig, + latestRestTestStatus, + restTestChat, + type RestTestRig +} from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' import { OrcaRuntimeService } from './orca-runtime' import { RpcDispatcher } from './rpc/dispatcher' import { SESSION_TAB_METHODS } from './rpc/methods/session-tabs' @@ -31,16 +32,20 @@ const CONTEXT = { clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] } -let rig: StartupRig +let rig: RestTestRig // Anything a test holds is let go before teardown, so a failed assertion cannot hang it. let releaseHeld = (): void => undefined beforeEach(async () => { - rig = await createStartupRig() + rig = await createRestTestRig() }) afterEach(async () => { releaseHeld() + // The listing starts the lease check and the restore without awaiting either; both finish before + // the rig removes their files. + await rig.host.reconcileRestartLeases() + await rig.host.restoreReadableSessions() setStructuredAgentSessionHost(null) await rig.dispose() closeTestJournalHostDatabases() @@ -93,14 +98,14 @@ describe('listing chat tabs at startup', () => { it('answers before any chat opens, then opens each one (T1)', async () => { const ids = ['session-1', 'session-2', 'session-3'] for (const sessionId of ids) { - await rig.chat(sessionId, { message: sessionId }) + await restTestChat(rig, sessionId, { message: sessionId }) } await rig.crash() await rig.boot() const opens = Promise.withResolvers() releaseHeld = opens.resolve let opened = 0 - rig.historyFilePath.mockImplementation(async () => { + rig.adapter.historyFilePath.mockImplementation(async () => { await opens.promise opened += 1 return null @@ -116,8 +121,8 @@ describe('listing chat tabs at startup', () => { }) it('answers while the startup lease check never finishes (T2)', async () => { - await rig.chat('session-1', { message: 'one' }) - await rig.chat('session-2', { message: 'two' }) + await restTestChat(rig, 'session-1', { message: 'one' }) + await restTestChat(rig, 'session-2', { message: 'two' }) await rig.crash() const probe = Promise.withResolvers() releaseHeld = probe.resolve @@ -137,7 +142,7 @@ describe('listing chat tabs at startup', () => { }) it('answers when the startup lease check fails (T2)', async () => { - await rig.chat('session-1', { message: 'one' }) + await restTestChat(rig, 'session-1', { message: 'one' }) await rig.crash() rig.probeOwner.mockRejectedValue(new Error('probe unavailable')) vi.spyOn(console, 'warn').mockImplementation(() => undefined) @@ -154,15 +159,18 @@ describe('listing chat tabs at startup', () => { ) }) - it('lists every chat when one fails to open, and the others still get status rows (T5)', async () => { - const ids = ['session-1', 'session-2', 'session-3'] + it('lists every chat when four fail to open, and the others still get status rows (T5)', async () => { + // More chats than the restore opens at once, with the first four failing: each failure must + // cost only its own chat, not one of the restore's four lanes. + const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5', 'session-6'] + const failing = ids.slice(0, 4) for (const sessionId of ids) { - await rig.chat(sessionId, { message: `asked ${sessionId}` }) + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) } await rig.crash() await rig.boot() - rig.historyFilePath.mockImplementation(async (sessionId) => { - if (sessionId === 'session-2') { + rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { + if (failing.includes(sessionId)) { throw new Error('EACCES: permission denied') } return null @@ -172,26 +180,31 @@ describe('listing chat tabs at startup', () => { const { listAll } = restartedRuntime() expect(await within(listAll())).toEqual(ids) - await vi.waitFor(() => { - expect(latestStatus(rig, 'session-1')).toMatchObject({ latestPrompt: 'asked session-1' }) - expect(latestStatus(rig, 'session-3')).toMatchObject({ latestPrompt: 'asked session-3' }) - }) - await vi.waitFor(() => - expect(warn).toHaveBeenCalledWith( - '[structured-agent-session] restoring chat history after listing failed', - expect.objectContaining({ message: 'EACCES: permission denied' }) + await restore.mock.results[0]?.value + for (const sessionId of ['session-5', 'session-6']) { + expect(rig.host.hasSession(sessionId)).toBe(true) + expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ + latestPrompt: `asked ${sessionId}` + }) + } + const failures = warn.mock.calls.filter( + ([message]) => message === '[structured-agent-session] restoring a chat for reading failed' + ) + expect(failures.map(([, detail]) => detail)).toEqual( + failing.map((sessionId) => + expect.objectContaining({ + sessionId, + error: expect.objectContaining({ message: 'EACCES: permission denied' }) + }) ) ) expect(await listAll()).toEqual(ids) expect(restore).toHaveBeenCalledOnce() - expect( - warn.mock.calls.filter(([message]) => String(message).includes('after listing')) - ).toHaveLength(1) }) it('lists in tab-table order, not the order the chats were created (T7)', async () => { for (const sessionId of ['session-c', 'session-a', 'session-b']) { - await rig.chat(sessionId, { listed: false }) + await restTestChat(rig, sessionId, { listed: false }) } for (const sessionId of ['session-b', 'session-c', 'session-a']) { await rig.store.setSessionTabVisibility(sessionId, true) @@ -205,7 +218,7 @@ describe('listing chat tabs at startup', () => { it('lists in the saved window order when the profile has no tab table (T7)', async () => { for (const sessionId of ['session-c', 'session-a', 'session-b']) { - await rig.chat(sessionId, { listed: false }) + await restTestChat(rig, sessionId, { listed: false }) } await rig.crash() const host = await rig.boot() From 855be2f8d619851481b4abb9d15f23b05327b371 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:54:31 -0700 Subject: [PATCH 008/107] fix(native-chat): start the chat history restore after the tab list has answered The restore that reads each chat's history was started, not awaited, from inside the listing call, so its synchronous start (about 110 ms on a 247-chat profile) ran on the listing's own stack and three chats opened before the list answered. It now starts on the next macrotask, after the answer has gone out. Also pins that listing writes nothing to the record store: tabs are projected from the store's own tab index, so a 247-tab list costs no per-tab transaction. --- ...tore-structured-agent-session-tabs-once.ts | 14 +++++-- ...runtime-structured-session-restore.test.ts | 9 ++++- ...-agent-session-startup-tab-restore.test.ts | 3 +- ...-agent-session-tab-listing-startup.test.ts | 37 +++++++++++++++++++ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 36539e4484ce..66779e93c6c6 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -91,10 +91,16 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const session of restored) { this.projectStructuredAgentSessionTab({ ...session, activate: false, notify: false }) } - // After the tabs are out, so no chat's history holds the list or paint; each chat's status row - // arrives as its history opens. One chat's failure leaves the others and the list alone. - void host?.restoreReadableSessions(listedIds).catch((error: unknown) => { - console.warn('[structured-agent-session] restoring chat history after listing failed', error) + // After the list's answer has gone out: the pass's synchronous start would otherwise run on the + // listing's stack. Each chat's status row arrives as its history opens; one chat's failure + // leaves the others and the list alone. + setImmediate(() => { + void host?.restoreReadableSessions(listedIds).catch((error: unknown) => { + console.warn( + '[structured-agent-session] restoring chat history after listing failed', + error + ) + }) }) const wasUnverifiable = this.structuredAgentSessionInventoryUnverifiable // No host means no one can say which chats exist; with none on disk, empty is the answer. diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 529a2a9ce42b..f566f8944a74 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -83,6 +83,9 @@ describe('structured session cold restoration', () => { const second = runtime.restoreStructuredAgentSessionTabs() expect(second).toBe(first) await Promise.all([first, second]) + // The history pass starts after the list's answer has gone out, never on its stack. + expect(restoreReadableSessions).not.toHaveBeenCalled() + await new Promise((resolve) => setImmediate(resolve)) expect(hydrate).toHaveBeenCalledWith('workspace-1', { allowAttachedWindow: true, @@ -146,7 +149,9 @@ describe('structured session cold restoration', () => { await runtime.restoreStructuredAgentSessionTabs() - expect(restoreReadableSessions).toHaveBeenCalledWith(['session-survives-rollback']) + await vi.waitFor(() => + expect(restoreReadableSessions).toHaveBeenCalledWith(['session-survives-rollback']) + ) }) it('treats an empty durable visible-session index as authoritative', async () => { @@ -200,7 +205,7 @@ describe('structured session cold restoration', () => { await runtime.restoreStructuredAgentSessionTabs() - expect(restoreReadableSessions).toHaveBeenCalledWith([]) + await vi.waitFor(() => expect(restoreReadableSessions).toHaveBeenCalledWith([])) }) it('normalizes a restored tab id and removes it when closed', async () => { diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index 79ad60a5486e..ec873be41502 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -217,8 +217,9 @@ function startupRuntime(options: { afterInstall?: () => void; profileChats?: str } } -/** The listing starts the history restore without awaiting it; this joins that pass. */ +/** The listing starts the history restore after its answer, without awaiting it; this joins it. */ async function historyRestored(): Promise { + await new Promise((resolve) => setImmediate(resolve)) await getStructuredAgentSessionHost()!.restoreReadableSessions() } diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 9c8687d524a0..4bf9f3952c58 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -12,6 +12,7 @@ import { restTestChat, type RestTestRig } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' +import { AgentSessionStoreTransactionQueue } from './agent-session-store-transaction-queue' import { OrcaRuntimeService } from './orca-runtime' import { RpcDispatcher } from './rpc/dispatcher' import { SESSION_TAB_METHODS } from './rpc/methods/session-tabs' @@ -45,6 +46,7 @@ afterEach(async () => { // The listing starts the lease check and the restore without awaiting either; both finish before // the rig removes their files. await rig.host.reconcileRestartLeases() + await new Promise((resolve) => setImmediate(resolve)) await rig.host.restoreReadableSessions() setStructuredAgentSessionHost(null) await rig.dispose() @@ -120,6 +122,40 @@ describe('listing chat tabs at startup', () => { expect(opened).toBe(3) }) + it('starts the history restore only after the list has answered', async () => { + for (const sessionId of ['session-1', 'session-2']) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + await rig.boot() + const restore = vi.spyOn(rig.host, 'restoreReadableSessions') + const { listAll } = restartedRuntime() + + expect(await listAll()).toEqual(['session-1', 'session-2']) + // The caller has the answer before the pass has started, let alone opened a chat. + expect(restore).not.toHaveBeenCalled() + expect(rig.adapter.historyFilePath).not.toHaveBeenCalled() + await vi.waitFor(() => expect(rig.host.hasSession('session-2')).toBe(true)) + expect(restore).toHaveBeenCalledOnce() + }) + + it('writes nothing to the record store while it lists', async () => { + const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + await rig.boot() + // Settled first, so a write here could only be the listing's own. + await rig.host.reconcileRestartLeases() + const writes = vi.spyOn(AgentSessionStoreTransactionQueue.prototype, 'transact') + const { listAll } = restartedRuntime() + + expect(await listAll()).toEqual(ids) + // Every write, a tab's visibility included, is one store transaction. + expect(writes).not.toHaveBeenCalled() + }) + it('answers while the startup lease check never finishes (T2)', async () => { await restTestChat(rig, 'session-1', { message: 'one' }) await restTestChat(rig, 'session-2', { message: 'two' }) @@ -180,6 +216,7 @@ describe('listing chat tabs at startup', () => { const { listAll } = restartedRuntime() expect(await within(listAll())).toEqual(ids) + await vi.waitFor(() => expect(restore).toHaveBeenCalledOnce()) await restore.mock.results[0]?.value for (const sessionId of ['session-5', 'session-6']) { expect(rig.host.hasSession(sessionId)).toBe(true) From bf3aebe58ea0bc7aae1501b4057d6dc66f010fb7 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:57:25 -0700 Subject: [PATCH 009/107] fix(native-chat): wait for the startup lease check again, and drop the per-write gate This PR had stopped startup waiting for the lease check (the pass that settles each chat's previous-run owner) and instead made every write to an unchecked chat wait for it. After #23964 that check can no longer fail startup, and a review run measured it at about 30 ms on a 247-chat Mac profile, so the gate bought nothing but a second mechanism. It also reordered two writes that arrived while a chat's lease flipped from unchecked to checked. Startup's structured step, the write path and worker release are back to the base's shape: startup awaits the non-throwing check, writes enter the chat's queue in arrival order, and an agent start still runs its own strict check before it acts. What stays is the listing from records and the tab table, and the chat history restore that starts after the list has answered. --- .../structured-agent-session-host.ts | 8 +- ...ctured-agent-session-refusal-retry.test.ts | 5 - ...uctured-agent-session-restart-reconcile.ts | 20 ---- ...ent-session-restored-status-parity.test.ts | 8 +- ...red-agent-session-write-lease-wait.test.ts | 111 ------------------ ...structured-agent-session-create-support.ts | 18 +-- ...runtime-structured-session-restore.test.ts | 2 +- .../worker/worker-release-completion.ts | 6 +- ...orker-release-structured-reconcile.test.ts | 62 ---------- .../structured-worker-tab-retirement.test.ts | 2 - ...ed-agent-session-startup-reconcile.test.ts | 11 +- ...-agent-session-startup-tab-restore.test.ts | 10 +- ...-agent-session-tab-listing-startup.test.ts | 46 +------- 13 files changed, 20 insertions(+), 289 deletions(-) delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts delete mode 100644 src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index f82b8a2cec15..b1ee31598231 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -10,10 +10,7 @@ import type { AgentJournalSnapshot } from '../../../shared/agent-session-journal import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' import type * as SessionWire from '../../../shared/agent-session-wire' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { - createRestartReconciler, - serializeAfterLeaseCheck -} from './structured-agent-session-restart-reconcile' +import { createRestartReconciler } from './structured-agent-session-restart-reconcile' import type { AgentSessionSubscribeInput } from './structured-agent-session-subscribers' import { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' import * as providerSupport from './structured-agent-session-provider-support' @@ -276,8 +273,7 @@ export class StructuredAgentSessionHost { publish: (sessionId, journal) => this.subscribers.publish(sessionId, journal), flushStreamedEvents: this.flushStreamedEvents, conversation: this.lifetime.conversation, - // Every write funnels through here: one to a chat with an unchecked lease waits for the check. - serialize: serializeAfterLeaseCheck(this.deps.store, this.reconcileLeases, this.serialize), + serialize: (sessionId, task) => this.serialize(sessionId, task), openConversation: this.conversationDelivery.open, ensureAgent: (sessionId) => ensureStructuredAgentSessionAgentForOperation(this.attachContext(), sessionId), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts index 7e0f530d179a..4c2db496c3cf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts @@ -335,14 +335,9 @@ describe('agentSessionRefusalOperationState host oracle', () => { ...current, lease: { ...current.lease, unreconciled: true } })) - // A write runs the lease check first; only one that cannot settle leaves this refusal. - const stuck = vi - .spyOn(reconciling.store, 'reconcileOnRestart') - .mockRejectedValue(new Error('execution_owner_reconciling')) const spec = { method, operationId: operationId() } record( await assertHostAgreement(reconciling, spec, 'execution_owner_reconciling', async () => { - stuck.mockRestore() await setLease(reconciling, (current) => ({ ...current, lease: { ...current.lease, unreconciled: false } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index b5c6111220af..23f0e0c6bfae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -107,26 +107,6 @@ function failureKey(failure: unknown): string { return String(failure) } -/** The chat's queue, entered by a write to a chat whose lease is still the previous run's only - * after the lease check, which startup does not wait for. The write waits for that check (one pass - * over every unreconciled lease, not only this chat's), or runs it. With none owed it enqueues at - * once, keeping its place. A check that fails gates nothing: a conversation write (a send, a Stop) - * is still admitted, a provider-child write is refused while the lease stays unreconciled, and an - * agent start reconciles again and refuses. */ -export function serializeAfterLeaseCheck( - store: Pick, - reconcile: (sessionId: string) => Promise, - serialize: (sessionId: string, task: () => Promise) => Promise -): (sessionId: string, task: () => Promise) => Promise { - return (sessionId, task) => - store.getRecord(sessionId)?.lease.unreconciled - ? reconcile(sessionId).then( - () => serialize(sessionId, task), - () => serialize(sessionId, task) - ) - : serialize(sessionId, task) -} - async function reconcileCurrentLeases(deps: { store: AgentSessionRecordStore probe: (record: AgentSessionRecord) => Promise diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts index ac399f72e8d8..333c1685051e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts @@ -53,19 +53,17 @@ it('ends every chat where a direct read of it lands, and shows no pre-crash work const readRig = await createRestTestRig({}, { root: readRoot }) rigs.push(readRig) - // Startup's shape: the lease check started, the list answered, then the pass. + // Startup's shape: the lease check, the list, then the pass. const passHost = await passRig.boot() - const passCheck = passHost.reconcileRestartLeases() + await passHost.reconcileRestartLeases() await passHost.restoreReadableSessions(CORPUS) - await passCheck // The same run, where the window reads each chat instead. const readHost = readRig.host - const readCheck = readHost.reconcileRestartLeases() + await readHost.reconcileRestartLeases() for (const sessionId of CORPUS) { await readHost.history({ sessionId, direction: 'tail' }) } - await readCheck // `updatedAt` is when the row was projected, a wall-clock read that differs between two boots. const rows = (rig: RestTestRig): Record> => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts deleted file mode 100644 index 18bb1714d92b..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lease-wait.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -// Startup no longer waits for the lease check, so a write can reach a chat whose lease the previous -// app run left unadjudicated. Such a write waits for the lease check (or runs it) before it acts, -// so while that check succeeds none is admitted against the old lease. - -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' -import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' -import { hostTestOperationId } from './structured-agent-session-host-test-data' -import { - createRestTestRig, - REST_TEST_CALLER, - restTestChat, - sendRestTestMessage, - type RestTestRig -} from './structured-agent-session-rest-test-rig' - -let rig: RestTestRig -// A failed assertion must not leave teardown waiting on a held probe. -let releaseProbe = (): void => undefined - -beforeEach(async () => { - rig = await createRestTestRig() -}) - -afterEach(async () => { - releaseProbe() - await rig.dispose() - closeTestJournalHostDatabases() - vi.restoreAllMocks() -}) - -/** Boots over a crashed run and starts the startup lease check with its owner probe held. */ -async function bootWithSlowLeaseCheck() { - await restTestChat(rig, 'session-a', { message: 'before the restart' }) - await rig.crash() - const probe = Promise.withResolvers() - const probing = Promise.withResolvers() - releaseProbe = probe.resolve - rig.probeOwner.mockImplementation(async () => { - probing.resolve() - await probe.promise - return { outcome: 'pid-absent' } - }) - const host = await rig.boot() - // What startup does now: start the check, and wait for nothing. - const startupCheck = host.reconcileRestartLeases() - await probing.promise - // Every admission, with the lease it was admitted against. - const admittedUnreconciled: boolean[] = [] - const admit = rig.store.admitMutationOperation - vi.spyOn(rig.store, 'admitMutationOperation').mockImplementation(async (args) => { - const admitted = await admit(args) - admittedUnreconciled.push(admitted?.record.lease.unreconciled ?? true) - return admitted - }) - return { host, probe, startupCheck, admittedUnreconciled } -} - -/** Long enough for an unheld write to open its chat and be admitted. */ -function settleTasks(): Promise { - return new Promise((resolve) => setTimeout(resolve, 200)) -} - -describe('a write during a slow startup lease check', () => { - it('lands a send after the chat’s check, and starts one agent (REQUIRED)', async () => { - const { probe, startupCheck, admittedUnreconciled } = await bootWithSlowLeaseCheck() - - let answered = false - const sent = sendRestTestMessage(rig, 'session-a', 'after the restart').then((result) => { - answered = true - return result - }) - await settleTasks() - expect(answered).toBe(false) - expect(admittedUnreconciled).toEqual([]) - - probe.resolve() - await expect(sent).resolves.toMatchObject({ ok: true }) - await startupCheck - await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) - - expect(admittedUnreconciled).toEqual([false]) - expect(rig.adapter.acquire).toHaveBeenCalledOnce() - }) - - it('holds a Stop until the chat’s check has run', async () => { - const { host, probe, startupCheck, admittedUnreconciled } = await bootWithSlowLeaseCheck() - - const fields = {} - const stopped = host.cancel(REST_TEST_CALLER, { - envelope: { - sessionId: 'session-a', - clientOperationId: hostTestOperationId(), - expectedRuntimeFence: null, - payloadFingerprint: computeAgentSessionPayloadFingerprint({ - method: 'agentSession.cancel', - sessionId: 'session-a', - fields - }) - } - }) - await settleTasks() - expect(admittedUnreconciled).toEqual([]) - - probe.resolve() - await expect(stopped).resolves.toMatchObject({ ok: true }) - await startupCheck - expect(admittedUnreconciled).toEqual([false]) - expect(rig.adapter.acquire).not.toHaveBeenCalled() - }) -}) diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index af7c4aa80d67..c0d4c596ffae 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -252,27 +252,16 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR return this.structuredAgentSessionTabRestorePromise } - /** Startup's structured step: the host and PTY inventory, awaited; the lease check, started and - * never awaited. Neither the tab list nor paint needs leases, and a write to a chat whose lease - * is unchecked first waits for that check (`serializeAfterLeaseCheck`). */ - async prepareStructuredAgentSessionStartupRestoration(): Promise { - await this.ensureStructuredAgentSessionInventory() - if (this.hasPersistedStructuredAgentSessionStore()) { - // Never rejects: a failure goes to the host's lease-reconcile report. - void getStructuredAgentSessionHost()?.reconcileRestartLeases() - } - } - - protected ensureStructuredAgentSessionInventory(): Promise { + prepareStructuredAgentSessionStartupRestoration(): Promise { this.structuredAgentSessionStartupRestorePromise ??= - this.ensureStructuredAgentSessionInventoryOnce().catch((error) => { + this.prepareStructuredAgentSessionStartupRestorationOnce().catch((error) => { this.structuredAgentSessionStartupRestorePromise = null throw error }) return this.structuredAgentSessionStartupRestorePromise } - private async ensureStructuredAgentSessionInventoryOnce(): Promise { + protected async prepareStructuredAgentSessionStartupRestorationOnce(): Promise { if (!this.hasPersistedStructuredAgentSessionStore()) { return } @@ -282,6 +271,7 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR this.ensureStructuredAgentSessionHost() ) await this.refreshMobileSessionPtyRecords() + await getStructuredAgentSessionHost()?.reconcileRestartLeases() } protected hasPersistedStructuredAgentSessionStore(): boolean { diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index f566f8944a74..4c116e13416d 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -51,7 +51,7 @@ describe('structured session cold restoration', () => { expect(restoreReadableSessions).not.toHaveBeenCalled() }) - it('loads records, inventories PTYs, starts the lease check, projects tabs, then restores history, once', async () => { + it('loads records, inventories PTYs, restores ownership, projects tabs, then restores history, once', async () => { const runtime = new OrcaRuntimeService() const hydrate = vi.fn() const refresh = vi.fn(async () => new Set()) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts index 09ebcdf3e617..4aca4d0420a0 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts @@ -19,7 +19,6 @@ import { workerTerminalLeaseIsCurrent } from './worker-terminal-release-lease' import { resolveStructuredWorkerForDispatch } from '../../orchestration-structured-worker-lifecycle' import { stopStructuredWorkerForRelease } from './structured-worker-release-stop' import { isStructuredWorkerHandle } from '../../../../structured-worker-identity' -import { getStructuredAgentSessionHost } from '../../../../../native-chat/agent-session-wire/structured-agent-session-registry' export { archiveSummary, @@ -96,9 +95,7 @@ async function completeWorkerTerminalReleaseOnce( if (isStructuredWorkerHandle(resource.terminal_handle)) { // Observation and archive capture both read the structured host, and after a restart nothing // has installed it yet — the startup recovery reconciler runs exactly this path. Installing it - // here is what lets the release see the session instead of reporting it unreadable. Startup no - // longer waits for the lease check either, and an unchecked worker lease from the previous run - // reads as live, so the release runs that check first (it never rejects; failures are reported). + // here is what lets the release see the session instead of reporting it unreadable. // // NOT yet handled, and deliberately follow-up: rebinding a restarted runtime to a structured // worker's redrive subscription. Until that exists, a worker that survives a restart has its @@ -110,7 +107,6 @@ async function completeWorkerTerminalReleaseOnce( error ) }) - await getStructuredAgentSessionHost()?.reconcileRestartLeases() } const worker = db.getWorkerDispatch(dispatchId) if (!worker || worker.agent_terminal_handle !== resource.terminal_handle) { diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts deleted file mode 100644 index 7972b485f7f1..000000000000 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-structured-reconcile.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -// After a restart the release is the first thing to read a structured worker, and startup no longer -// waits for the lease check. The release runs that check before it observes the worker (T6). - -import { afterEach, describe, expect, it, vi } from 'vitest' -import { setStructuredAgentSessionHost } from '../../../../../native-chat/agent-session-wire/structured-agent-session-registry' -import { mintStructuredWorkerHandle } from '../../../../structured-worker-identity' - -const { order } = vi.hoisted(() => { - const order: string[] = [] - return { order } -}) - -vi.mock('./worker-observation', () => ({ - inspectWorkerTerminal: vi.fn(async () => { - order.push('observe') - return { status: 'identity_changed' } - }) -})) - -const { completeWorkerTerminalRelease } = await import('./worker-release-completion') - -afterEach(() => { - order.length = 0 - setStructuredAgentSessionHost(null) -}) - -describe('structured worker release after a restart', () => { - it('reads the worker only once the restart reconcile has settled its lease', async () => { - const handle = mintStructuredWorkerHandle() - const runtime = { - ensureStructuredAgentSessionHost: vi.fn(async () => { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the release reads only the reconcile stubbed here. - setStructuredAgentSessionHost({ - reconcileRestartLeases: async () => { - // A reconcile that probes processes settles after a macrotask, not a microtask. - await new Promise((resolve) => setTimeout(resolve, 0)) - order.push('reconcile') - } - } as never) - }) - } - const db = { - getWorkerDispatch: () => ({ agent_terminal_handle: handle }), - revertWorkerTerminalReleaseToRetained: () => ({ archive_source: null, archive_status: null }), - recordWorkerTerminalRecoveryAttempt: vi.fn() - } - - const receipt = await completeWorkerTerminalRelease({ - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this path reads only the runtime member stubbed above. - runtime: runtime as never, - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retained path reads only the dispatch rows stubbed above. - db: db as never, - dispatchId: 'dispatch-1', - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retained path reads only the id and handle. - resource: { id: 'resource-1', terminal_handle: handle } as never, - mode: 'recovery' - }) - - expect(receipt.state).toBe('retained') - expect(order).toEqual(['reconcile', 'observe']) - }) -}) diff --git a/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts b/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts index 0945d32834f0..b48dccf52a39 100644 --- a/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts +++ b/src/main/runtime/rpc/methods/structured-worker-tab-retirement.test.ts @@ -43,11 +43,9 @@ function installHost(options: { closeThrows?: boolean; lease?: Record undefined, hasSession: () => attached, hold: async () => {}, release: () => {}, diff --git a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts index a759341ab2d8..33b26f7569bb 100644 --- a/src/main/runtime/structured-agent-session-startup-reconcile.test.ts +++ b/src/main/runtime/structured-agent-session-startup-reconcile.test.ts @@ -83,8 +83,7 @@ it('finishes startup over records a newer Orca wrote, reports it, and writes not // What the renderer's startup awaits through `app:prepareTerminalStartupRestoration`. await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).resolves.toBeUndefined() - // Startup does not wait for the check; its failure is reported when it lands. - await vi.waitFor(() => expect(onError).toHaveBeenCalledOnce()) + expect(onError).toHaveBeenCalledOnce() expect(onError).toHaveBeenCalledWith({ scope: 'structured-agent-session-lease-reconcile', error: expect.objectContaining({ message: 'agent_session_legacy_required' }) @@ -104,10 +103,8 @@ it('logs the failure when the host has no error sink', async () => { startupRuntime().prepareStructuredAgentSessionStartupRestoration() ).resolves.toBeUndefined() - await vi.waitFor(() => - expect(warn).toHaveBeenCalledWith( - '[structured-agent-session] reconciling chat leases failed', - expect.objectContaining({ message: 'agent_session_legacy_required' }) - ) + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] reconciling chat leases failed', + expect.objectContaining({ message: 'agent_session_legacy_required' }) ) }) diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index dae8a776e195..50d23f0d09d7 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -297,15 +297,13 @@ describe('restoring the chat tabs open at quit', () => { await seedStore(records, { visible: chats }) await seedHistory(records) vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const { runtime, onError, published } = startupRuntime({ + const { runtime, published } = startupRuntime({ afterInstall: () => { lock.failing = true } }) await runtime.prepareStructuredAgentSessionStartupRestoration() - // Startup starts the lease check without awaiting it; its report is where it lands. - await vi.waitFor(() => expect(onError).toHaveBeenCalledOnce()) const prepared = lock.refused await runtime.restoreStructuredAgentSessionTabs() await historyRestored() @@ -329,8 +327,6 @@ describe('restoring the chat tabs open at quit', () => { const writes = spyOnTabWrites() const { runtime, published } = startupRuntime({ profileChats: [CHAT_A, CHAT_B] }) await runtime.prepareStructuredAgentSessionStartupRestoration() - // Joins startup's un-awaited lease check, so it takes no lock grant meant for the seed. - await getStructuredAgentSessionHost()!.reconcileRestartLeases() // One more take, then held: a seed written chat by chat would stop part way. lock.grants = 1 @@ -350,7 +346,7 @@ describe('restoring the chat tabs open at quit', () => { await seedStore(records) await seedHistory(records) vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const { runtime, onError, published } = startupRuntime({ + const { runtime, published } = startupRuntime({ profileChats: [CHAT_A, CHAT_B], afterInstall: () => { lock.failing = true @@ -358,7 +354,6 @@ describe('restoring the chat tabs open at quit', () => { }) await runtime.prepareStructuredAgentSessionStartupRestoration() - await vi.waitFor(() => expect(onError).toHaveBeenCalledOnce()) const prepared = lock.refused await runtime.restoreStructuredAgentSessionTabs() await historyRestored() @@ -376,7 +371,6 @@ describe('restoring the chat tabs open at quit', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) const { runtime, published } = startupRuntime({ profileChats: [CHAT_A, CHAT_B] }) await runtime.prepareStructuredAgentSessionStartupRestoration() - await getStructuredAgentSessionHost()!.reconcileRestartLeases() lock.failing = true await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 4bf9f3952c58..078be7f491a0 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -1,5 +1,5 @@ // The chat tab list a client asks for at startup answers from records and the tab table. It waits -// for no chat's history and for no lease check; each chat's history opens after the answer. +// for no chat's history; each chat's history opens after the answer. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../shared/protocol-version' @@ -43,9 +43,8 @@ beforeEach(async () => { afterEach(async () => { releaseHeld() - // The listing starts the lease check and the restore without awaiting either; both finish before - // the rig removes their files. - await rig.host.reconcileRestartLeases() + // The listing starts the history restore after its answer, without awaiting it; it finishes + // before the rig removes its files. await new Promise((resolve) => setImmediate(resolve)) await rig.host.restoreReadableSessions() setStructuredAgentSessionHost(null) @@ -156,45 +155,6 @@ describe('listing chat tabs at startup', () => { expect(writes).not.toHaveBeenCalled() }) - it('answers while the startup lease check never finishes (T2)', async () => { - await restTestChat(rig, 'session-1', { message: 'one' }) - await restTestChat(rig, 'session-2', { message: 'two' }) - await rig.crash() - const probe = Promise.withResolvers() - releaseHeld = probe.resolve - rig.probeOwner.mockImplementation(async () => { - await probe.promise - return { outcome: 'pid-absent' } - }) - await rig.boot() - const { runtime, listAll } = restartedRuntime() - - // What the renderer's startup awaits through `app:prepareTerminalStartupRestoration`. - expect(await within(runtime.prepareStructuredAgentSessionStartupRestoration())).toBe(undefined) - expect(await within(listAll())).toEqual(['session-1', 'session-2']) - expect(rig.probeOwner).toHaveBeenCalled() - probe.resolve() - await rig.host.reconcileRestartLeases() - }) - - it('answers when the startup lease check fails (T2)', async () => { - await restTestChat(rig, 'session-1', { message: 'one' }) - await rig.crash() - rig.probeOwner.mockRejectedValue(new Error('probe unavailable')) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const onLeaseReconcileFailure = vi.fn() - await rig.boot({ onLeaseReconcileFailure }) - const { runtime, listAll } = restartedRuntime() - - await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).resolves.toBe(undefined) - expect(await within(listAll())).toEqual(['session-1']) - await vi.waitFor(() => - expect(onLeaseReconcileFailure).toHaveBeenCalledWith( - expect.objectContaining({ message: 'probe unavailable' }) - ) - ) - }) - it('lists every chat when four fail to open, and the others still get status rows (T5)', async () => { // More chats than the restore opens at once, with the first four failing: each failure must // cost only its own chat, not one of the restore's four lanes. From ebdc5b92a010af0e05e9d84f3c8771fecd0f1ef9 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:18:24 -0700 Subject: [PATCH 010/107] fix(native-chat): start the chat history restore once the tab list's answer is out The history restore was deferred with setImmediate from inside the tab restore, on the theory that the list answer would be sent by then. It is not: after the tab restore, session.tabs.listAll and the subscribe streams still run the PTY census (daemon, git and SSH calls), and the deferred restore fired during it. Four or five chats opened before the list answered, which is what the deferral was meant to prevent. The tab restore now only records the history restore it owes. Every session.tabs handler that restores tabs (list, listAll, subscribe, subscribeAll) goes through one wrapper that starts the owed restore when that handler's answer has settled: sent, failed or abandoned. The first to settle starts it; the rest find it started. The tab restore itself is no longer callable from a handler without that wrapper. Also: both quit checks in the restore now have a test (one skips the lease bookkeeping, one the open), and a comment no longer says a chat with missing history explains itself; it reads empty. --- .../structured-agent-session-host-tabs.ts | 4 +- ...gent-session-restore-after-listing.test.ts | 30 +++++- ...tore-structured-agent-session-tabs-once.ts | 32 +++--- src/main/runtime/orca-runtime-runtime-id.ts | 3 + ...runtime-structured-session-restore.test.ts | 7 +- .../rpc/methods/session-tabs-inventory.ts | 39 +++----- src/main/runtime/rpc/methods/session-tabs.ts | 99 ++++++++++--------- .../methods/structured-session-tab-restore.ts | 21 +++- ...-agent-session-startup-tab-restore.test.ts | 12 ++- ...-agent-session-tab-listing-startup.test.ts | 73 ++++++++++---- 10 files changed, 209 insertions(+), 111 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts index 0facc7cb037e..61d3d353fe23 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts @@ -48,8 +48,8 @@ export type StructuredAgentSessionTab = { } /** Tabs from durable state alone: each requested id, once, in the order given, that has a record - * this host serves. Opens nothing: a chat whose history is missing or unreadable keeps its tab and - * explains itself when read. */ + * this host serves. Opens nothing: a chat whose history is unreadable keeps its tab and its read + * says why; one whose history is missing keeps its tab and reads empty. */ export function listPersistedSessionTabs( deps: { store: { getRecord: (sessionId: string) => AgentSessionRecord | null } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts index 9752089002b9..7fe8dac70a21 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts @@ -68,8 +68,35 @@ describe('the restore pass after the tab list', () => { expect(host.hasSession('session-1')).toBe(true) }) - it('opens nothing more once the host is quitting', async () => { + it('does not open a chat it reached before quit began but opens after', async () => { + await restTestChat(rig, LAST, { message: 'asked' }) + await rig.crash() + const host = await rig.boot() + const { runtimeState, lifetime } = host.collaboratorsForTests() + // Holds the chat between the pass reaching it and its open. + const settling = Promise.withResolvers() + releaseHeld = settling.resolve + const resolveRecovery = runtimeState.resolveRecovery.bind(runtimeState) + const recovery = vi.spyOn(runtimeState, 'resolveRecovery').mockImplementation(async (id) => { + await settling.promise + return resolveRecovery(id) + }) + const pass = host.restoreReadableSessions([LAST]) + await vi.waitFor(() => expect(recovery).toHaveBeenCalled()) + + const teardown = host.flushAllStreamedEvents() + await vi.waitFor(() => expect(lifetime.isDisposed()).toBe(true)) + settling.resolve() + await teardown + await pass + + expect(restTestOpens(rig, LAST)).toBe(0) + expect(host.hasSession(LAST)).toBe(false) + }) + + it('opens nothing more, and settles no more leases, once the host is quitting', async () => { const { host, pass, release } = await bootWithHeldPass() + const recovery = vi.spyOn(host.collaboratorsForTests().runtimeState, 'resolveRecovery') const teardown = host.flushAllStreamedEvents() release() @@ -78,5 +105,6 @@ describe('the restore pass after the tab list', () => { expect(restTestOpens(rig, LAST)).toBe(0) expect(host.hasSession(LAST)).toBe(false) + expect(recovery).not.toHaveBeenCalledWith(LAST) }) }) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index ef33e0316db6..ef11b8717651 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -91,17 +91,17 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const session of restored) { this.projectStructuredAgentSessionTab({ ...session, activate: false, notify: false }) } - // After the list's answer has gone out: the pass's synchronous start would otherwise run on the - // listing's stack. Each chat's status row arrives as its history opens; one chat's failure - // leaves the others and the list alone. - setImmediate(() => { - void host?.restoreReadableSessions(listedIds).catch((error: unknown) => { - console.warn( - '[structured-agent-session] restoring chat history after listing failed', - error - ) - }) - }) + // Each chat's status row arrives as its history opens; one chat's failure leaves the others and + // the list alone. Whoever answers with this list starts it, once that answer is out. + this.owedStructuredAgentSessionHistoryRestore = host + ? () => + void host.restoreReadableSessions(listedIds).catch((error: unknown) => { + console.warn( + '[structured-agent-session] restoring chat history after listing failed', + error + ) + }) + : null const wasUnverifiable = this.structuredAgentSessionInventoryUnverifiable // No host means no one can say which chats exist; with none on disk, empty is the answer. this.structuredAgentSessionInventoryUnverifiable = @@ -112,6 +112,16 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu } } + /** Starts the history restore the tab restore owes, once. On the next macrotask, so a caller that + * starts it as it answers has sent that answer first. */ + startStructuredAgentSessionHistoryRestore(): void { + const owed = this.owedStructuredAgentSessionHistoryRestore + this.owedStructuredAgentSessionHistoryRestore = null + if (owed) { + setImmediate(owed) + } + } + async publishStructuredAgentSessionTab(input: { workspaceId: string sessionId: string diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index f033cb456ebc..1ff91e49b43b 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -121,6 +121,9 @@ export class OrcaRuntimeWithRuntimeId { protected structuredAgentSessionTabRestorePromise: Promise | null = null + // The history restore a tab restore owes, until a caller that answered with its list starts it. + protected owedStructuredAgentSessionHistoryRestore: (() => void) | null = null + // Whether the last tab restore ran with chats on disk but no host to list them. protected structuredAgentSessionInventoryUnverifiable = false diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 4c116e13416d..880cf09e68cf 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -83,7 +83,10 @@ describe('structured session cold restoration', () => { const second = runtime.restoreStructuredAgentSessionTabs() expect(second).toBe(first) await Promise.all([first, second]) - // The history pass starts after the list's answer has gone out, never on its stack. + // The history pass is owed, not started: the caller that answers with the list starts it. + expect(restoreReadableSessions).not.toHaveBeenCalled() + runtime.startStructuredAgentSessionHistoryRestore() + runtime.startStructuredAgentSessionHistoryRestore() expect(restoreReadableSessions).not.toHaveBeenCalled() await new Promise((resolve) => setImmediate(resolve)) @@ -148,6 +151,7 @@ describe('structured session cold restoration', () => { } as never) await runtime.restoreStructuredAgentSessionTabs() + runtime.startStructuredAgentSessionHistoryRestore() await vi.waitFor(() => expect(restoreReadableSessions).toHaveBeenCalledWith(['session-survives-rollback']) @@ -204,6 +208,7 @@ describe('structured session cold restoration', () => { } as never) await runtime.restoreStructuredAgentSessionTabs() + runtime.startStructuredAgentSessionHistoryRestore() await vi.waitFor(() => expect(restoreReadableSessions).toHaveBeenCalledWith([])) }) diff --git a/src/main/runtime/rpc/methods/session-tabs-inventory.ts b/src/main/runtime/rpc/methods/session-tabs-inventory.ts index 3ab06a298fc6..5ed2ee8d902b 100644 --- a/src/main/runtime/rpc/methods/session-tabs-inventory.ts +++ b/src/main/runtime/rpc/methods/session-tabs-inventory.ts @@ -6,7 +6,7 @@ import { projectSessionTabAgentStatus } from './session-tab-agent-status-project import { projectSessionTabBrowserPlacements } from './session-tab-browser-placement-projection' import { createSessionTabsRetirementProofDelta } from './session-tabs-retirement-proof-delta' import { isStructuredNativeChatEnabled } from './structured-agent-session-policy' -import { restoreStructuredTabsIfSupported } from './structured-session-tab-restore' +import { answerAfterStructuredTabRestore } from './structured-session-tab-restore' type SessionTabsInventory = { snapshots: RuntimeMobileSessionTabsResult[] @@ -242,37 +242,30 @@ export async function subscribeSessionTabsInventory( } let collected: Awaited> | undefined try { - // Why: restore after registering, so an unsubscribe or socket close while it runs still finds the stream. - const restoring = restoreStructuredTabsIfSupported(context) - if (restoring) { - await restoring - if (closed) { - return - } - } - for (let attempt = 1; !collected; attempt += 1) { - censusInvalidated = false - const candidate = await collectSessionTabsInventory( - { ...context, signal: inventoryController.signal }, - true - ) - if (closed) { - return - } - if (censusInvalidated) { + // Why: restore after registering, so an unsubscribe or socket close while it runs still finds the + // stream. The snapshots emit right after this settles, before the history restore it starts. + collected = await answerAfterStructuredTabRestore(context, async () => { + for (let attempt = 1; !closed; attempt += 1) { + censusInvalidated = false + const candidate = await collectSessionTabsInventory( + { ...context, signal: inventoryController.signal }, + true + ) + if (!censusInvalidated || closed) { + return candidate + } clearBufferedChanges() if (attempt === MAX_CENSUS_COLLECTION_ATTEMPTS) { throw new Error('session_tabs_inventory_unstable') } - } else { - collected = candidate } - } + return undefined + }) } catch (error) { runtime.cleanupSubscription(subscriptionId) throw error } - if (closed) { + if (closed || !collected) { return } const { inventory, changeSequence } = collected diff --git a/src/main/runtime/rpc/methods/session-tabs.ts b/src/main/runtime/rpc/methods/session-tabs.ts index 8d3d1af33bf2..e3199ad0e12e 100644 --- a/src/main/runtime/rpc/methods/session-tabs.ts +++ b/src/main/runtime/rpc/methods/session-tabs.ts @@ -15,7 +15,7 @@ import { import { SESSION_TAB_MARKDOWN_METHODS } from './session-tab-markdown-methods' import { SESSION_TAB_MUTATION_METHODS } from './session-tab-mutation-methods' import { createSessionTabsRetirementProofDelta } from './session-tabs-retirement-proof-delta' -import { restoreStructuredTabsIfSupported } from './structured-session-tab-restore' +import { answerAfterStructuredTabRestore } from './structured-session-tab-restore' import { isStructuredNativeChatEnabled } from './structured-agent-session-policy' import { assertLegacyAiVaultResumeCommandAllowed } from '../../../ai-vault/structured-session-ownership' import { SessionTabsUnsubscribeAllParams } from '../../../../shared/rpc-contract/session-tabs-params' @@ -25,23 +25,21 @@ export const SESSION_TAB_METHODS = [ defineMethod({ name: 'session.tabs.list', params: WorktreeTabSelector, - handler: async (params, { runtime, pairedDeviceId, clientKind, clientCapabilities }) => { - await restoreStructuredTabsIfSupported({ runtime, clientKind, clientCapabilities }) - return projectSessionTabsForClient( - await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId), - clientKind, - clientCapabilities, - isStructuredNativeChatEnabled(runtime) + handler: (params, { runtime, pairedDeviceId, clientKind, clientCapabilities }) => + answerAfterStructuredTabRestore({ runtime, clientKind, clientCapabilities }, async () => + projectSessionTabsForClient( + await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId), + clientKind, + clientCapabilities, + isStructuredNativeChatEnabled(runtime) + ) ) - } }), defineMethod({ name: 'session.tabs.listAll', params: null, - handler: async (_params, context) => { - await restoreStructuredTabsIfSupported(context) - return listSessionTabsInventory(context) - } + handler: (_params, context) => + answerAfterStructuredTabRestore(context, () => listSessionTabsInventory(context)) }), ...SESSION_TAB_MUTATION_METHODS, ...SESSION_TAB_CLOSE_METHODS, @@ -144,51 +142,56 @@ export const SESSION_TAB_METHODS = [ register(explicitWorktreeId) } try { - await restoreStructuredTabsIfSupported({ runtime, clientKind, clientCapabilities }) - if (released) { - return - } - const initial = await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId) - if (released) { - return - } - if (!subscriptionId) { - register(initial.worktree) - if (released) { - return - } - } - const subscribedWorktree = initial.worktree - const withProofDelta = createSessionTabsRetirementProofDelta(clientCapabilities) - emit({ - type: 'snapshot', - ...withProofDelta( - projectSessionTabsForClient( - initial, - clientKind, - clientCapabilities, - isStructuredNativeChatEnabled(runtime) - ) - ) - }) - if (released) { - return - } - stopListening = runtime.onMobileSessionTabsChanged((snapshot) => { - if (snapshot.worktree === subscribedWorktree) { + // The first snapshot emits inside this, before the history restore it starts. + await answerAfterStructuredTabRestore( + { runtime, clientKind, clientCapabilities }, + async () => { + if (released) { + return + } + const initial = await runtime.listMobileSessionTabs(params.worktree, pairedDeviceId) + if (released) { + return + } + if (!subscriptionId) { + register(initial.worktree) + if (released) { + return + } + } + const subscribedWorktree = initial.worktree + const withProofDelta = createSessionTabsRetirementProofDelta(clientCapabilities) emit({ - type: 'updated', + type: 'snapshot', ...withProofDelta( projectSessionTabsForClient( - snapshot, + initial, clientKind, clientCapabilities, isStructuredNativeChatEnabled(runtime) ) ) }) + if (released) { + return + } + stopListening = runtime.onMobileSessionTabsChanged((snapshot) => { + if (snapshot.worktree === subscribedWorktree) { + emit({ + type: 'updated', + ...withProofDelta( + projectSessionTabsForClient( + snapshot, + clientKind, + clientCapabilities, + isStructuredNativeChatEnabled(runtime) + ) + ) + }) + } + }, pairedDeviceId) } - }, pairedDeviceId) + ) } catch (error) { // A stream already ended by its release must not also report an error. if (released) { diff --git a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts index 6ec962414229..06c79eb20f7a 100644 --- a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts +++ b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts @@ -11,7 +11,7 @@ import { * project after a desktop restart — no chat and no prompt. The setting still gates it, because * with structured chat off there is nothing for any mobile client to reach. Restoring spawns no * provider child for a cleanly closed session. */ -export function restoreStructuredTabsIfSupported( +function restoreStructuredTabsIfSupported( context: Pick ): Promise | undefined { const shouldRestore = @@ -24,3 +24,22 @@ export function restoreStructuredTabsIfSupported( // Nothing to restore: callers skip the await, so a stream's setup keeps its timing. return undefined } + +/** The tab restore, then the caller's answer. The chat history restore the tab restore owes starts + * only once that answer has settled (sent, failed or abandoned), so the answer's own work, such as + * the PTY census, never waits behind it. The first caller to settle starts it. */ +export async function answerAfterStructuredTabRestore( + context: Pick, + answer: () => Promise +): Promise { + const restoring = restoreStructuredTabsIfSupported(context) + if (!restoring) { + return answer() + } + try { + await restoring + return await answer() + } finally { + context.runtime.startStructuredAgentSessionHistoryRestore?.() + } +} diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index 50d23f0d09d7..38e127dcf845 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -217,8 +217,10 @@ function startupRuntime(options: { afterInstall?: () => void; profileChats?: str } } -/** The listing starts the history restore after its answer, without awaiting it; this joins it. */ -async function historyRestored(): Promise { +/** Starts the history restore the listing owes, as the caller that answers with it does, and joins + * it. */ +async function historyRestored(runtime: OrcaRuntimeService): Promise { + runtime.startStructuredAgentSessionHistoryRestore() await new Promise((resolve) => setImmediate(resolve)) await getStructuredAgentSessionHost()!.restoreReadableSessions() } @@ -263,7 +265,7 @@ describe('restoring the chat tabs open at quit', () => { }) await expect(runtime.restoreStructuredAgentSessionTabs()).resolves.toBeUndefined() - await historyRestored() + await historyRestored(runtime) expect(published().map((tab) => tab.id)).toEqual([ `agent-session:${CHAT_A}`, @@ -306,7 +308,7 @@ describe('restoring the chat tabs open at quit', () => { await runtime.prepareStructuredAgentSessionStartupRestoration() const prepared = lock.refused await runtime.restoreStructuredAgentSessionTabs() - await historyRestored() + await historyRestored(runtime) expect(published()).toHaveLength(count) expect(prepared).toBe(1) @@ -356,7 +358,7 @@ describe('restoring the chat tabs open at quit', () => { await runtime.prepareStructuredAgentSessionStartupRestoration() const prepared = lock.refused await runtime.restoreStructuredAgentSessionTabs() - await historyRestored() + await historyRestored(runtime) expect(published()).toHaveLength(2) expect(prepared).toBe(1) diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 078be7f491a0..2823e29a618d 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -25,6 +25,7 @@ type ListingInternals = { getKnownWorkspaceSessionWorktreeIds(): Set hydrateHeadlessMobileSessionTabsFromWorkspaceSession(): Set refreshMobileSessionPtyRecords(): Promise | null> + refreshMobileSessionPtyInventory(): Promise ensureStructuredAgentSessionHost(): Promise } @@ -66,6 +67,11 @@ function restartedRuntime(workspaceSession: unknown = null) { internal.getKnownWorkspaceSessionWorktreeIds = () => new Set() internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() + // The census an answer takes after the tab restore: daemon, git and SSH round trips in the app. + internal.refreshMobileSessionPtyInventory = async () => { + await new Promise((resolve) => setTimeout(resolve, 20)) + return null + } internal.ensureStructuredAgentSessionHost = async () => { setStructuredAgentSessionHost(rig.host) } @@ -80,11 +86,31 @@ function restartedRuntime(workspaceSession: unknown = null) { } // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: listAll answers with the inventory shape its handler returns. const { snapshots } = response.result as { snapshots: RuntimeMobileSessionTabsResult[] } - return snapshots.flatMap((snapshot) => - snapshot.tabs.flatMap((tab) => (tab.type === 'agent-session' ? [tab.sessionId] : [])) - ) + return chatIds(snapshots) } - return { runtime, listAll } + /** Mobile's stream: resolves with its first snapshots, and what had run when they were sent. */ + const subscribeAll = (seen: () => unknown) => + new Promise<{ ids: string[]; atAnswer: unknown }>((resolve, reject) => { + void dispatcher.dispatchStreaming( + { id: 'subscribe-all', authToken: 'tok', method: 'session.tabs.subscribeAll' }, + (message) => { + const frame = JSON.parse(message) + if (!frame.ok) { + reject(new Error(JSON.stringify(frame.error))) + } else if (frame.result?.type === 'snapshots') { + resolve({ ids: chatIds(frame.result.snapshots), atAnswer: seen() }) + } + }, + { ...CONTEXT, connectionId: 'connection-1' } + ) + }) + return { runtime, listAll, subscribeAll } +} + +function chatIds(snapshots: RuntimeMobileSessionTabsResult[]): string[] { + return snapshots.flatMap((snapshot) => + snapshot.tabs.flatMap((tab) => (tab.type === 'agent-session' ? [tab.sessionId] : [])) + ) } /** Resolves with the answer, or with 'still waiting' once `ms` pass without one. */ @@ -121,22 +147,31 @@ describe('listing chat tabs at startup', () => { expect(opened).toBe(3) }) - it('starts the history restore only after the list has answered', async () => { - for (const sessionId of ['session-1', 'session-2']) { - await restTestChat(rig, sessionId, { message: sessionId }) - } - await rig.crash() - await rig.boot() - const restore = vi.spyOn(rig.host, 'restoreReadableSessions') - const { listAll } = restartedRuntime() + it.each(['listAll', 'subscribeAll'] as const)( + 'starts the history restore only after %s has answered, census and all', + async (method) => { + const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + await rig.boot() + const restore = vi.spyOn(rig.host, 'restoreReadableSessions') + const { runtime, listAll, subscribeAll } = restartedRuntime() + const seen = () => ({ + started: restore.mock.calls.length, + opens: rig.adapter.historyFilePath.mock.calls.length + }) - expect(await listAll()).toEqual(['session-1', 'session-2']) - // The caller has the answer before the pass has started, let alone opened a chat. - expect(restore).not.toHaveBeenCalled() - expect(rig.adapter.historyFilePath).not.toHaveBeenCalled() - await vi.waitFor(() => expect(rig.host.hasSession('session-2')).toBe(true)) - expect(restore).toHaveBeenCalledOnce() - }) + const answered = + method === 'listAll' ? { ids: await listAll(), atAnswer: seen() } : await subscribeAll(seen) + + expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0 } }) + await vi.waitFor(() => ids.forEach((id) => expect(rig.host.hasSession(id)).toBe(true))) + expect(restore).toHaveBeenCalledOnce() + runtime.cleanupSubscriptionsForConnection('connection-1') + } + ) it('writes nothing to the record store while it lists', async () => { const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] From c3af299b326e7a9619365c1b9c151b9027b635ad Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:46:00 -0700 Subject: [PATCH 011/107] test(native-chat): give the rest rig's and T3's dispatch waits a 10 s timeout Both waited on the provider dispatch with vi.waitFor's default 1 s, and T3's wait failed once under load. Neighbouring structured-session tests wait 10 s for the same kind of assertion. --- .../structured-agent-session-persisted-tab-listing.test.ts | 2 +- .../structured-agent-session-rest-test-rig.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts index b75ab410111e..13fd635be312 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts @@ -65,7 +65,7 @@ describe('the tab list after a restart', () => { expect(page.ok && page.page.items).toEqual([]) const sent = await sendRestTestMessage(rig, 'session-new', 'first words') expect(sent).toMatchObject({ ok: true }) - await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledOnce(), { timeout: 10_000 }) const after = await host.history({ sessionId: 'session-new', direction: 'tail' }) expect(JSON.stringify(after.ok && after.page)).toContain('first words') }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts index 6ae31f023cf0..1f5fdad403d1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts @@ -283,7 +283,8 @@ export async function restTestChat( if (options.message === undefined) { return } - const dispatched = rig.adapter.dispatch.mock.calls.length + const { dispatch } = rig.adapter + const dispatched = dispatch.mock.calls.length const sent = await rig.host.send( REST_TEST_CALLER, restTestSend(options.message, attached.fence, sessionId) @@ -291,7 +292,9 @@ export async function restTestChat( if (!sent.ok) { throw new Error(`send refused: ${sent.refusal.code}`) } - await vi.waitFor(() => expect(rig.adapter.dispatch.mock.calls.length).toBeGreaterThan(dispatched)) + await vi.waitFor(() => expect(dispatch.mock.calls.length).toBeGreaterThan(dispatched), { + timeout: 10_000 + }) } /** Creates the rig's own chat, lists its tab, and sends one message so its journal is on disk. */ From 0cbeb1441ffc451af9744c1691d40b7b84b7cd76 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:55:28 -0700 Subject: [PATCH 012/107] fix(native-chat): derive /clear replacements once per tab listing, not once per tab Every tab snapshot the runtime stores or answers re-applies the /clear replacements, and deriving them walks every chat record. The listing stores one snapshot per restored tab and answers one per worktree, so a 247-tab, 1,301-record profile derived them 289 times per listAll. On main those derivations were split up by the per-tab store writes this PR removed, so here they ran as one long main-process task (about 1.9 s on the Windows measurement machine). Each session.tabs listing request, tab restore included, now derives them at most once, on first use, and every snapshot it stores or answers reuses that value. Anything outside a listing, and any timer or task a listing leaves running after it answers, still derives them fresh. A /clear committed while a listing waits on I/O is still shown: its own commit re-projects the stored snapshot. --- ...e-prune-mobile-session-tab-group-layout.ts | 4 +- ...tore-structured-agent-session-tabs-once.ts | 3 +- src/main/runtime/orca-runtime-runtime-id.ts | 4 +- .../methods/structured-session-tab-restore.ts | 21 +++++--- ...-agent-session-tab-listing-startup.test.ts | 45 ++++++++++++++++ ...ed-conversation-replacements-scope.test.ts | 54 +++++++++++++++++++ ...uctured-conversation-replacements-scope.ts | 33 ++++++++++++ 7 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 src/main/runtime/structured-conversation-replacements-scope.test.ts create mode 100644 src/main/runtime/structured-conversation-replacements-scope.ts diff --git a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts index 75005726c2db..f59ce30e1aa2 100644 --- a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts +++ b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts @@ -23,8 +23,8 @@ import type { import { buildRuntimeMobileAgentStatus } from './runtime-mobile-agent-status-builder' import { FIRST_PANE_ID } from '../../shared/pane-key' import { isTerminalLeafId, makePaneKey, parsePaneKey } from '../../shared/stable-pane-id' -import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' +import { currentConversationReplacements } from './structured-conversation-replacements-scope' import { resolveStructuredWorkerAuthority } from './structured-worker-authority' import { structuredWorkerAgentStatus } from './orchestration/structured-worker-group-addressing' @@ -78,7 +78,7 @@ export class OrcaRuntimeWithPruneMobileSessionTabGroupLayout extends OrcaRuntime protected toMobileSessionTabsResult( snapshot: RuntimeMobileSessionTabsSnapshot ): RuntimeMobileSessionTabsResult { - for (const replacement of getStructuredAgentSessionHost()?.conversationReplacements?.() ?? []) { + for (const replacement of currentConversationReplacements()) { snapshot = replaceConversationInSnapshot(snapshot, replacement) } return projectRuntimeMobileSessionTabs(snapshot, this.getMobileSessionProjectionHost()) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index ef11b8717651..c075ef76863e 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -6,6 +6,7 @@ import { replaceConversationInSnapshot } from './structured-conversation-tab-rep import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { seedStructuredAgentSessionTabIndex } from './structured-agent-session-tab-index-seed' +import { currentConversationReplacements } from './structured-conversation-replacements-scope' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { RuntimeMobileSessionAgentTab, @@ -85,7 +86,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu restored.map((session) => session.sessionId) ) // Past the seed, projecting records nothing. - for (const replacement of host?.conversationReplacements?.() ?? []) { + for (const replacement of currentConversationReplacements()) { this.replaceStructuredAgentSessionTab(replacement) } for (const session of restored) { diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index 1ff91e49b43b..515b2ead6167 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -1,8 +1,8 @@ // @ts-nocheck -- mechanically split from OrcaRuntimeService; behavior is covered by AST equivalence and characterization tests. import { randomUUID } from 'node:crypto' import { preserveTerminalRetirementProofs } from './mobile-session-terminal-retirement-proof' -import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' +import { currentConversationReplacements } from './structured-conversation-replacements-scope' import type { TuiAgent } from '../../shared/tui-agent' import type { RuntimeStore } from './runtime-store-contract' import type { RuntimeClientSettingsController } from './runtime-client-settings' @@ -105,7 +105,7 @@ export class OrcaRuntimeWithRuntimeId { worktreeId: string, snapshot: RuntimeMobileSessionTabsSnapshot ): RuntimeMobileSessionTabsSnapshot { - for (const replacement of getStructuredAgentSessionHost()?.conversationReplacements?.() ?? []) { + for (const replacement of currentConversationReplacements()) { snapshot = replaceConversationInSnapshot(snapshot, replacement) } const existing = this.mobileSessionTabsByWorktree.get(worktreeId) diff --git a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts index 06c79eb20f7a..0afba5215883 100644 --- a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts +++ b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts @@ -1,4 +1,5 @@ import type { RpcContext } from '../core' +import { withListingConversationReplacements } from '../../structured-conversation-replacements-scope' import { isStructuredNativeChatEnabled, supportsStructuredAgentSessions @@ -32,14 +33,20 @@ export async function answerAfterStructuredTabRestore( context: Pick, answer: () => Promise ): Promise { - const restoring = restoreStructuredTabsIfSupported(context) - if (!restoring) { - return answer() - } + let restoring: Promise | undefined try { - await restoring - return await answer() + // One listing, tab restore included, so the /clear replacements every snapshot it stores or + // answers applies are derived once for it, not once per tab and per worktree. + return await withListingConversationReplacements(async () => { + restoring = restoreStructuredTabsIfSupported(context) + if (restoring) { + await restoring + } + return answer() + }) } finally { - context.runtime.startStructuredAgentSessionHistoryRestore?.() + if (restoring) { + context.runtime.startStructuredAgentSessionHistoryRestore?.() + } } } diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 2823e29a618d..524fa570f7b4 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -173,6 +173,51 @@ describe('listing chat tabs at startup', () => { } ) + it.each(['listAll', 'subscribeAll'] as const)( + 'derives the /clear replacements once per %s, whatever the tab and worktree count', + async (method) => { + const ids = Array.from({ length: 9 }, (_, index) => `session-${index + 1}`) + for (const [index, sessionId] of ids.entries()) { + await restTestChat(rig, sessionId, { workspaceId: `workspace-${(index % 3) + 1}` }) + } + await rig.crash() + await rig.boot() + const derived = vi.spyOn(rig.host, 'conversationReplacements') + const { runtime, listAll, subscribeAll } = restartedRuntime() + const answer = async () => + method === 'listAll' ? listAll() : (await subscribeAll(() => undefined)).ids + + // The first answer also restores the tabs; the second only lists them. + for (const _listing of [1, 2]) { + derived.mockClear() + expect((await answer()).toSorted()).toEqual(ids.toSorted()) + expect(derived).toHaveBeenCalledOnce() + runtime.cleanupSubscriptionsForConnection('connection-1') + } + } + ) + + it('answers with a /clear committed after an earlier listing', async () => { + await restTestChat(rig, 'session-cleared') + await restTestChat(rig, 'session-fresh', { listed: false }) + await rig.crash() + await rig.boot() + const { listAll } = restartedRuntime() + expect(await listAll()).toEqual(['session-cleared']) + + const record = rig.store.getRecord('session-cleared')! + await rig.store.setConversationCommand('session-cleared', record.lease.runtimeFence, { + command: 'clear', + state: 'completed', + phase: 'committed', + operationId: 'clear-1', + callerKey: 'client-1', + replacementSessionId: 'session-fresh' + }) + + expect(await listAll()).toEqual(['session-fresh']) + }) + it('writes nothing to the record store while it lists', async () => { const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] for (const sessionId of ids) { diff --git a/src/main/runtime/structured-conversation-replacements-scope.test.ts b/src/main/runtime/structured-conversation-replacements-scope.test.ts new file mode 100644 index 000000000000..5302fe498b14 --- /dev/null +++ b/src/main/runtime/structured-conversation-replacements-scope.test.ts @@ -0,0 +1,54 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { + currentConversationReplacements, + withListingConversationReplacements +} from './structured-conversation-replacements-scope' + +afterEach(() => { + setStructuredAgentSessionHost(null) +}) + +function hostDeriving() { + const conversationReplacements = vi.fn(() => []) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the scope reads only this member. + setStructuredAgentSessionHost({ conversationReplacements } as never) + return conversationReplacements +} + +it('derives once inside a listing, across its awaits', async () => { + const derive = hostDeriving() + + await withListingConversationReplacements(async () => { + currentConversationReplacements() + await new Promise((resolve) => setTimeout(resolve, 1)) + currentConversationReplacements() + }) + + expect(derive).toHaveBeenCalledOnce() +}) + +it('derives on every call outside a listing', () => { + const derive = hostDeriving() + + currentConversationReplacements() + currentConversationReplacements() + + expect(derive).toHaveBeenCalledTimes(2) +}) + +it('derives fresh in work a listing started that outlives it', async () => { + const derive = hostDeriving() + const later = Promise.withResolvers() + + await withListingConversationReplacements(async () => { + currentConversationReplacements() + setTimeout(() => { + currentConversationReplacements() + later.resolve() + }, 5) + }) + await later.promise + + expect(derive).toHaveBeenCalledTimes(2) +}) diff --git a/src/main/runtime/structured-conversation-replacements-scope.ts b/src/main/runtime/structured-conversation-replacements-scope.ts new file mode 100644 index 000000000000..161abaf328e5 --- /dev/null +++ b/src/main/runtime/structured-conversation-replacements-scope.ts @@ -0,0 +1,33 @@ +// Every stored or answered tab snapshot re-applies the /clear replacements, and deriving them walks +// every record. A listing stores and answers once per tab and per worktree, so inside one listing +// they are derived once and shared; everywhere else each call derives them fresh. + +import { AsyncLocalStorage } from 'node:async_hooks' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' + +type ListingScope = { replacements?: readonly ConversationReplacement[]; ended: boolean } + +const listingScope = new AsyncLocalStorage() + +/** Runs one listing request with its replacements derived at most once. */ +export async function withListingConversationReplacements(run: () => Promise): Promise { + const scope: ListingScope = { ended: false } + try { + return await listingScope.run(scope, run) + } finally { + // A timer or task the listing started outlives it and must derive fresh. + scope.ended = true + } +} + +export function currentConversationReplacements(): readonly ConversationReplacement[] { + const scope = listingScope.getStore() + const host = getStructuredAgentSessionHost() + if (!scope || scope.ended) { + return host?.conversationReplacements?.() ?? [] + } + // Only a host's answer is kept: before the listing installs one there is nothing to derive yet. + scope.replacements ??= host?.conversationReplacements?.() + return scope.replacements ?? [] +} From 4768d53505b41d9ac2a4d821d1cff6888749dfe3 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:14:35 -0700 Subject: [PATCH 013/107] fix(native-chat): derive /clear replacements right before each listing loop The previous commit kept one derived set of /clear replacements for a whole listing request, reused across the listing's later waits (the PTY census, the authoritative re-collection, subscribeAll's retries). A chat revealed from its history during one of those waits was then answered as its replacement: the stale set still renamed its tab. The request-wide scope is gone. The two loops that carry the cost each derive the set immediately before they run, with nothing awaited in between: the tab restore's projection loop (one snapshot per tab) and the collection's per-worktree answer loop, after its census. Both hand the set to storeMobileSessionSnapshot / toMobileSessionTabsResult through an optional parameter; every other caller derives fresh, as before. A first listing now derives twice (one per loop), later ones once, independent of tab and worktree count. --- ...-mobile-visible-graph-changed-worktrees.ts | 5 +- ...e-prune-mobile-session-tab-group-layout.ts | 10 +++- ...tore-structured-agent-session-tabs-once.ts | 12 ++-- src/main/runtime/orca-runtime-runtime-id.ts | 10 +++- ...runtime-structured-session-restore.test.ts | 4 +- .../methods/structured-session-tab-restore.ts | 21 +++---- ...-agent-session-tab-listing-startup.test.ts | 59 +++++++++++++++++-- ...ed-conversation-replacements-scope.test.ts | 54 ----------------- ...uctured-conversation-replacements-scope.ts | 33 ----------- 9 files changed, 91 insertions(+), 117 deletions(-) delete mode 100644 src/main/runtime/structured-conversation-replacements-scope.test.ts delete mode 100644 src/main/runtime/structured-conversation-replacements-scope.ts diff --git a/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts b/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts index 391a33f0e247..3ffa30b8a3a7 100644 --- a/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts +++ b/src/main/runtime/orca-runtime-collect-mobile-visible-graph-changed-worktrees.ts @@ -4,6 +4,7 @@ import type { RuntimeMobileSessionTabsResult, RuntimeSyncedTab } from '../../sha import type { RuntimeLeafRecord } from './runtime-terminal-state-records' import type { PtyControllerInventory } from './runtime-pty-controller-contract' import { parseExecutionHostId } from '../../shared/execution-host' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' export class OrcaRuntimeWithCollectMobileVisibleGraphChangedWorktrees extends OrcaRuntimeWithSyncWindowGraph { // Why: toMobileSessionTabsResult resolves handles/titles from this.tabs and @@ -100,9 +101,11 @@ export class OrcaRuntimeWithCollectMobileVisibleGraphChangedWorktrees extends Or this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession() const ptyInventory = await this.refreshMobileSessionPtyInventory() this.restoreLivePairedRendererSessionOwnedMobileTerminals(null) + // Once for every worktree's answer, after the census's await, so a change made during it counts. + const replacements = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? [] const snapshots = [...this.mobileSessionTabsByWorktree.values()].map((snapshot) => this.projectMobileSessionTabsForClient( - this.toMobileSessionTabsResult(snapshot), + this.toMobileSessionTabsResult(snapshot, replacements), clientNavigationId ) ) diff --git a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts index f59ce30e1aa2..e88643b03401 100644 --- a/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts +++ b/src/main/runtime/orca-runtime-prune-mobile-session-tab-group-layout.ts @@ -24,7 +24,8 @@ import { buildRuntimeMobileAgentStatus } from './runtime-mobile-agent-status-bui import { FIRST_PANE_ID } from '../../shared/pane-key' import { isTerminalLeafId, makePaneKey, parsePaneKey } from '../../shared/stable-pane-id' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' -import { currentConversationReplacements } from './structured-conversation-replacements-scope' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import { resolveStructuredWorkerAuthority } from './structured-worker-authority' import { structuredWorkerAgentStatus } from './orchestration/structured-worker-group-addressing' @@ -76,9 +77,12 @@ export class OrcaRuntimeWithPruneMobileSessionTabGroupLayout extends OrcaRuntime /** Transforms an internal mobile session tab snapshot into a sanitized client payload, resolving launch-agent ownership and normalizing titles. */ protected toMobileSessionTabsResult( - snapshot: RuntimeMobileSessionTabsSnapshot + snapshot: RuntimeMobileSessionTabsSnapshot, + // A caller answering many worktrees in one synchronous loop derives these once for it. + replacements: readonly ConversationReplacement[] = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? + [] ): RuntimeMobileSessionTabsResult { - for (const replacement of currentConversationReplacements()) { + for (const replacement of replacements) { snapshot = replaceConversationInSnapshot(snapshot, replacement) } return projectRuntimeMobileSessionTabs(snapshot, this.getMobileSessionProjectionHost()) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index c075ef76863e..672cb9313574 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -6,7 +6,6 @@ import { replaceConversationInSnapshot } from './structured-conversation-tab-rep import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { seedStructuredAgentSessionTabIndex } from './structured-agent-session-tab-index-seed' -import { currentConversationReplacements } from './structured-conversation-replacements-scope' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { RuntimeMobileSessionAgentTab, @@ -86,11 +85,14 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu restored.map((session) => session.sessionId) ) // Past the seed, projecting records nothing. - for (const replacement of currentConversationReplacements()) { + // Derived once for the loop below, which stores a snapshot per tab; nothing awaits in between. + const replacements = host?.conversationReplacements?.() ?? [] + for (const replacement of replacements) { this.replaceStructuredAgentSessionTab(replacement) } + const quiet = { activate: false, notify: false, replacements } for (const session of restored) { - this.projectStructuredAgentSessionTab({ ...session, activate: false, notify: false }) + this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } // Each chat's status row arrives as its history opens; one chat's failure leaves the others and // the list alone. Whoever answers with this list starts it, once that answer is out. @@ -152,6 +154,8 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu activate: boolean notify?: boolean replacesSessionId?: string + /** The /clear replacements, derived once by a caller projecting many tabs in one loop. */ + replacements?: readonly ConversationReplacement[] }): void { const existing = this.mobileSessionTabsByWorktree.get(input.workspaceId) const id = `agent-session:${input.sessionId}` @@ -228,7 +232,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu ...(existing?.tabGroupLayout ? { tabGroupLayout: existing.tabGroupLayout } : {}), tabs } - const stored = this.storeMobileSessionSnapshot(input.workspaceId, snapshot) + const stored = this.storeMobileSessionSnapshot(input.workspaceId, snapshot, input.replacements) if (input.notify !== false) { this.emitMobileSessionTabsSnapshot(stored) } diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index 515b2ead6167..18178c6ef935 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -2,7 +2,8 @@ import { randomUUID } from 'node:crypto' import { preserveTerminalRetirementProofs } from './mobile-session-terminal-retirement-proof' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' -import { currentConversationReplacements } from './structured-conversation-replacements-scope' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import type { TuiAgent } from '../../shared/tui-agent' import type { RuntimeStore } from './runtime-store-contract' import type { RuntimeClientSettingsController } from './runtime-client-settings' @@ -103,9 +104,12 @@ export class OrcaRuntimeWithRuntimeId { /** Single host writer for mobile session snapshots; versions are total-order stamps. */ protected storeMobileSessionSnapshot( worktreeId: string, - snapshot: RuntimeMobileSessionTabsSnapshot + snapshot: RuntimeMobileSessionTabsSnapshot, + // A caller storing many snapshots in one synchronous loop derives these once for it. + replacements: readonly ConversationReplacement[] = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? + [] ): RuntimeMobileSessionTabsSnapshot { - for (const replacement of currentConversationReplacements()) { + for (const replacement of replacements) { snapshot = replaceConversationInSnapshot(snapshot, replacement) } const existing = this.mobileSessionTabsByWorktree.get(worktreeId) diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 880cf09e68cf..78f6a91c7f5d 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -367,7 +367,9 @@ describe('structured session cold restoration', () => { sessionId: 'restored-claude', agent: 'claude', activate: false, - notify: false + notify: false, + // Derived once by the restore for every tab it projects. + replacements: [] }) const restored = await runtime.listMobileSessionTabs('id:workspace-1') diff --git a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts index 0afba5215883..06c79eb20f7a 100644 --- a/src/main/runtime/rpc/methods/structured-session-tab-restore.ts +++ b/src/main/runtime/rpc/methods/structured-session-tab-restore.ts @@ -1,5 +1,4 @@ import type { RpcContext } from '../core' -import { withListingConversationReplacements } from '../../structured-conversation-replacements-scope' import { isStructuredNativeChatEnabled, supportsStructuredAgentSessions @@ -33,20 +32,14 @@ export async function answerAfterStructuredTabRestore( context: Pick, answer: () => Promise ): Promise { - let restoring: Promise | undefined + const restoring = restoreStructuredTabsIfSupported(context) + if (!restoring) { + return answer() + } try { - // One listing, tab restore included, so the /clear replacements every snapshot it stores or - // answers applies are derived once for it, not once per tab and per worktree. - return await withListingConversationReplacements(async () => { - restoring = restoreStructuredTabsIfSupported(context) - if (restoring) { - await restoring - } - return answer() - }) + await restoring + return await answer() } finally { - if (restoring) { - context.runtime.startStructuredAgentSessionHistoryRestore?.() - } + context.runtime.startStructuredAgentSessionHistoryRestore?.() } } diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 524fa570f7b4..cb58ce96de87 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -174,7 +174,7 @@ describe('listing chat tabs at startup', () => { ) it.each(['listAll', 'subscribeAll'] as const)( - 'derives the /clear replacements once per %s, whatever the tab and worktree count', + 'derives the /clear replacements once per loop of %s, whatever the tab and worktree count', async (method) => { const ids = Array.from({ length: 9 }, (_, index) => `session-${index + 1}`) for (const [index, sessionId] of ids.entries()) { @@ -187,11 +187,12 @@ describe('listing chat tabs at startup', () => { const answer = async () => method === 'listAll' ? listAll() : (await subscribeAll(() => undefined)).ids - // The first answer also restores the tabs; the second only lists them. - for (const _listing of [1, 2]) { + // The first answer also restores the tabs (one loop over every tab, one over every worktree); + // the second only lists them. + for (const loops of [2, 1]) { derived.mockClear() expect((await answer()).toSorted()).toEqual(ids.toSorted()) - expect(derived).toHaveBeenCalledOnce() + expect(derived).toHaveBeenCalledTimes(loops) runtime.cleanupSubscriptionsForConnection('connection-1') } } @@ -218,6 +219,56 @@ describe('listing chat tabs at startup', () => { expect(await listAll()).toEqual(['session-fresh']) }) + it('answers with a cleared chat revealed while the listing waits on its census', async () => { + await restTestChat(rig, 'session-cleared') + await restTestChat(rig, 'session-fresh', { listed: false }) + await rig.crash() + await rig.boot() + const { runtime, listAll } = restartedRuntime() + expect(await listAll()).toEqual(['session-cleared']) + const record = rig.store.getRecord('session-cleared')! + await rig.store.setConversationCommand('session-cleared', record.lease.runtimeFence, { + command: 'clear', + state: 'completed', + phase: 'committed', + operationId: 'clear-1', + callerKey: 'client-1', + replacementSessionId: 'session-fresh' + }) + expect(await listAll()).toEqual(['session-fresh']) + // An authoritative census that could not reach every host collects a second time; hold that one. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these runtime members exist; they are protected. + const internal = runtime as unknown as { + supportsAuthoritativeSessionTabsInventory(): boolean + getAuthoritativeSessionTabsInventoryEpoch(): string + refreshMobileSessionPtyInventory(): Promise + } + internal.supportsAuthoritativeSessionTabsInventory = () => true + internal.getAuthoritativeSessionTabsInventoryEpoch = () => 'epoch-1' + let collections = 0 + const inSecond = Promise.withResolvers() + const held = Promise.withResolvers() + releaseHeld = held.resolve + internal.refreshMobileSessionPtyInventory = async () => { + collections += 1 + if (collections === 2) { + inSecond.resolve() + await held.promise + } + return null + } + const listing = listAll() + await inSecond.promise + + // agentSession.reveal from another client: the cleared chat opened from its history. + const revealed = await rig.host.revealSession('session-cleared') + await runtime.publishStructuredAgentSessionTab({ ...revealed, activate: true }) + held.resolve() + + expect(await listing).toEqual(['session-cleared']) + expect(await listAll()).toEqual(['session-cleared']) + }) + it('writes nothing to the record store while it lists', async () => { const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5'] for (const sessionId of ids) { diff --git a/src/main/runtime/structured-conversation-replacements-scope.test.ts b/src/main/runtime/structured-conversation-replacements-scope.test.ts deleted file mode 100644 index 5302fe498b14..000000000000 --- a/src/main/runtime/structured-conversation-replacements-scope.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { afterEach, expect, it, vi } from 'vitest' -import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' -import { - currentConversationReplacements, - withListingConversationReplacements -} from './structured-conversation-replacements-scope' - -afterEach(() => { - setStructuredAgentSessionHost(null) -}) - -function hostDeriving() { - const conversationReplacements = vi.fn(() => []) - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the scope reads only this member. - setStructuredAgentSessionHost({ conversationReplacements } as never) - return conversationReplacements -} - -it('derives once inside a listing, across its awaits', async () => { - const derive = hostDeriving() - - await withListingConversationReplacements(async () => { - currentConversationReplacements() - await new Promise((resolve) => setTimeout(resolve, 1)) - currentConversationReplacements() - }) - - expect(derive).toHaveBeenCalledOnce() -}) - -it('derives on every call outside a listing', () => { - const derive = hostDeriving() - - currentConversationReplacements() - currentConversationReplacements() - - expect(derive).toHaveBeenCalledTimes(2) -}) - -it('derives fresh in work a listing started that outlives it', async () => { - const derive = hostDeriving() - const later = Promise.withResolvers() - - await withListingConversationReplacements(async () => { - currentConversationReplacements() - setTimeout(() => { - currentConversationReplacements() - later.resolve() - }, 5) - }) - await later.promise - - expect(derive).toHaveBeenCalledTimes(2) -}) diff --git a/src/main/runtime/structured-conversation-replacements-scope.ts b/src/main/runtime/structured-conversation-replacements-scope.ts deleted file mode 100644 index 161abaf328e5..000000000000 --- a/src/main/runtime/structured-conversation-replacements-scope.ts +++ /dev/null @@ -1,33 +0,0 @@ -// Every stored or answered tab snapshot re-applies the /clear replacements, and deriving them walks -// every record. A listing stores and answers once per tab and per worktree, so inside one listing -// they are derived once and shared; everywhere else each call derives them fresh. - -import { AsyncLocalStorage } from 'node:async_hooks' -import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' -import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' - -type ListingScope = { replacements?: readonly ConversationReplacement[]; ended: boolean } - -const listingScope = new AsyncLocalStorage() - -/** Runs one listing request with its replacements derived at most once. */ -export async function withListingConversationReplacements(run: () => Promise): Promise { - const scope: ListingScope = { ended: false } - try { - return await listingScope.run(scope, run) - } finally { - // A timer or task the listing started outlives it and must derive fresh. - scope.ended = true - } -} - -export function currentConversationReplacements(): readonly ConversationReplacement[] { - const scope = listingScope.getStore() - const host = getStructuredAgentSessionHost() - if (!scope || scope.ended) { - return host?.conversationReplacements?.() ?? [] - } - // Only a host's answer is kept: before the listing installs one there is nothing to derive yet. - scope.replacements ??= host?.conversationReplacements?.() - return scope.replacements ?? [] -} From 316d7cc60fec8c62ad9ae30c61431365dd1eecc4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:28:30 -0700 Subject: [PATCH 014/107] fix(native-chat): restore chat history after the tab list one chat at a time The history restore that now runs after the tab list ran four chats at once. Its work is synchronous on the main thread (each open folds the chat's journal), so four lanes add no throughput: they only put up to four fold chunks in every event-loop turn, and each step of a user's read or send waits behind a whole turn. On the Mac measurement rig the first send after a restart waited 2.4-7.4 s behind the restore. It now opens one chat at a time. One chat's failure still costs only that chat, and a chat closed, or a host quitting, while it runs is still skipped. --- .../structured-agent-session-restart-restore.test.ts | 11 +++++++---- .../structured-agent-session-restart-restore.ts | 4 +++- ...ctured-agent-session-restore-after-listing.test.ts | 8 ++++---- ...ructured-agent-session-tab-listing-startup.test.ts | 3 +-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 7540e8159744..8a376b4e0ae9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -35,7 +35,7 @@ const NO_OPEN_DEPS = { describe('restart journal restoration', () => { beforeEach(() => restoreRead.mockReset()) - it('bounds historical journal parsing to four sessions at a time', async () => { + it('opens one chat at a time', async () => { const gate = Promise.withResolvers() let active = 0 let peak = 0 @@ -71,13 +71,16 @@ describe('restart journal restoration', () => { onReadable: () => undefined }) - await vi.waitFor(() => expect(active).toBe(4)) - expect(restoreRead).toHaveBeenCalledTimes(4) + await vi.waitFor(() => expect(active).toBeGreaterThan(0)) + // Held long enough for a second lane, had there been one, to have started. + await new Promise((resolve) => setTimeout(resolve, 20)) + const startedWhileHeld = restoreRead.mock.calls.length gate.resolve() await restoration + expect(startedWhileHeld).toBe(1) expect(restoreRead).toHaveBeenCalledTimes(records.length) - expect(peak).toBe(4) + expect(peak).toBe(1) }) it('lets the event loop run between chats', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index fd621db47cbe..ffbf7dfae22a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -21,7 +21,9 @@ import type { } from './structured-agent-session-conversation-open' import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' -const JOURNAL_RESTORE_CONCURRENCY = 4 +// One chat at a time: the restore is CPU-bound on the main thread, so more lanes only lengthen each +// event-loop turn that a user's read or send waits behind. +const JOURNAL_RESTORE_CONCURRENCY = 1 export type StructuredAgentSessionReadRestoreDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts index 7fe8dac70a21..09ac84c75f5d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts @@ -27,8 +27,8 @@ afterEach(async () => { vi.restoreAllMocks() }) -/** Boots over a crashed run and starts the pass with the first four opens held, so it has not - * reached the last chat. */ +/** Boots over a crashed run and starts the pass with its first open held, so it has not reached + * the last chat. */ async function bootWithHeldPass() { for (const sessionId of IDS) { await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) @@ -38,13 +38,13 @@ async function bootWithHeldPass() { const held = Promise.withResolvers() releaseHeld = held.resolve rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { - if (sessionId !== LAST) { + if (sessionId === IDS[0]) { await held.promise } return null }) const pass = host.restoreReadableSessions(IDS) - await vi.waitFor(() => expect(rig.adapter.historyFilePath).toHaveBeenCalledTimes(4)) + await vi.waitFor(() => expect(rig.adapter.historyFilePath).toHaveBeenCalledOnce()) return { host, pass, release: held.resolve } } diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index cb58ce96de87..6400a61903ae 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -287,8 +287,7 @@ describe('listing chat tabs at startup', () => { }) it('lists every chat when four fail to open, and the others still get status rows (T5)', async () => { - // More chats than the restore opens at once, with the first four failing: each failure must - // cost only its own chat, not one of the restore's four lanes. + // The first four failing: each failure must cost only its own chat, never the chats after it. const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5', 'session-6'] const failing = ids.slice(0, 4) for (const sessionId of ids) { From 1df09d5e6bfeb447788932d93c3000bce6be2392 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:15:13 -0700 Subject: [PATCH 015/107] fix(native-chat): keep a throwing failure sink from failing the startup chat read The lease bookkeeping failure reporter called the host's failure sink directly, so a sink that threw turned a reported, recoverable store failure back into a rejected startup reconcile or read restore. The reporter now catches a sink throw and logs both the original failure and the sink error with console.warn. --- ...uctured-agent-session-restart-reconcile.ts | 10 +++++++- ...-session-startup-reconcile-failure.test.ts | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index 23f0e0c6bfae..f349f252ebed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -62,7 +62,15 @@ export function reportEachFailureOnce( const key = failureKey(failure) if (key !== reported) { reported = key - onFailure?.(failure) + try { + onFailure?.(failure) + } catch (sinkError) { + // A throwing sink must not turn the reported failure back into a failed read. + console.warn('[structured-agent-session] reporting a lease bookkeeping failure failed', { + failure, + sinkError + }) + } } }, clear: () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts index 984475c4dd71..1d7b6621b3ef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts @@ -194,3 +194,27 @@ it('restores a chat for reading when resolving its recovery cannot write the sto expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() expect(onLeaseReconcileFailure).toHaveBeenCalledWith(expect.objectContaining({ code: 'ELOCKED' })) }) + +it.each([ + ['startup reconcile', (host: StructuredAgentSessionHost) => host.reconcileRestartLeases()], + ['read restore', (host: StructuredAgentSessionHost) => host.restoreReadableSessions([SESSION])] +])('keeps the %s resolving when the failure sink throws', async (_step, read) => { + const { host, onLeaseReconcileFailure } = await relaunch() + const sinkError = new Error('error sink failed') + onLeaseReconcileFailure.mockImplementation(() => { + throw sinkError + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + lock.failing = true + try { + await expect(read(host)).resolves.toBeUndefined() + + expect(onLeaseReconcileFailure).toHaveBeenCalledOnce() + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('reporting a lease bookkeeping failure failed'), + expect.objectContaining({ failure: expect.objectContaining({ code: 'ELOCKED' }), sinkError }) + ) + } finally { + warn.mockRestore() + } +}) From 0a8bf771e7257b557c41876cb9920ec264c531e4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:44:09 -0700 Subject: [PATCH 016/107] test(native-chat): a send during a slow startup lease check starts its agent once, after the check Startup waits for the lease check (which settles each chat's owner from the previous app run) only on the desktop window's path. A phone, the CLI or a remote host's peer can send before it has finished, and the check is not awaited on any RPC write. This pins what keeps that safe. The send is accepted into the chat at once, and its delivery's agent start runs the same check itself, so it waits for the one already in flight. Two sends made while the check is held start one agent, after the check, and both are delivered. Without the agent start's own check, both messages are rejected ("couldn't restart", reason: the lease is still being checked) and no agent starts. --- ...on-send-during-startup-lease-check.test.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts new file mode 100644 index 000000000000..e293bec65e7c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts @@ -0,0 +1,82 @@ +// A client that is not the desktop window (a phone, the CLI, a remote host's peer) can send before +// startup's lease check has settled the previous run's owner. The send is accepted into the chat +// at once; the agent it needs starts only after the check, through the agent start's own check. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { + createRestTestRig, + restTestChat, + sendRestTestMessage, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const CHAT = 'session-a' + +let rig: RestTestRig +// A failed assertion must not leave teardown waiting on a held probe. +let releaseProbe = (): void => undefined + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + releaseProbe() + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +describe('a send during a slow startup lease check', () => { + it('is accepted at once, and its agent starts once, after the check', async () => { + await restTestChat(rig, CHAT, { message: 'before the restart' }) + await rig.crash() + const probe = Promise.withResolvers() + const probing = Promise.withResolvers() + releaseProbe = probe.resolve + let checked = false + rig.probeOwner.mockImplementation(async () => { + probing.resolve() + await probe.promise + checked = true + return { outcome: 'pid-absent' } + }) + const host = await rig.boot() + const startupCheck = host.reconcileRestartLeases() + await probing.promise + // Each start, with whether the check had adjudicated the chat's lease by then. + const starts: { checked: boolean; unreconciled: boolean | undefined }[] = [] + const acquire = rig.adapter.acquire.getMockImplementation() + rig.adapter.acquire.mockImplementation(async (input) => { + starts.push({ checked, unreconciled: rig.store.getRecord(CHAT)?.lease.unreconciled }) + if (!acquire) { + throw new Error('the rig acquires through its own implementation') + } + return acquire(input) + }) + + const first = await sendRestTestMessage(rig, CHAT, 'first after the restart') + expect(first, JSON.stringify(first)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + // Queued behind the first one's delivery, which is waiting on the check. + const second = sendRestTestMessage(rig, CHAT, 'second after the restart') + // Long enough for a delivery that did not wait to have started its agent. + await new Promise((resolve) => setTimeout(resolve, 200)) + expect(starts).toEqual([]) + expect(rig.adapter.dispatch).not.toHaveBeenCalled() + + probe.resolve() + await startupCheck + expect(await second, 'second send').toMatchObject({ ok: true }) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2), { + timeout: 10_000 + }) + + expect(starts).toEqual([{ checked: true, unreconciled: false }]) + const submissions = (await host.journalSnapshot(CHAT)).submissions.slice(-2) + expect(submissions.map((entry) => entry.dispatchState)).toEqual(['accepted', 'accepted']) + }) +}) From 531c031ea7bf6628a0c6d11fa06920010d7e9d68 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:39:25 -0700 Subject: [PATCH 017/107] fix(native-chat): keep the runtime-id module within its 400-line limit The runtime module size test failed in CI: orca-runtime-runtime-id.ts had grown to 402 lines. The field for the owed chat-history restore moves to the module whose restoreStructuredAgentSessionTabs owns the tab restore, and an import this PR had reordered goes back to main's order. --- ...ca-runtime-get-structured-agent-session-create-support.ts | 3 +++ src/main/runtime/orca-runtime-runtime-id.ts | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index c0d4c596ffae..b242208ad127 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -26,6 +26,9 @@ import { parseWorkspaceKey } from '../../shared/workspace-scope' import { applyStructuredCodexWorkspaceTrust } from '../agent-workspace-trust-spawn' export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaRuntimeWithGetWorktreePs { + // The history restore a tab restore owes, until a caller that answered with its list starts it. + protected owedStructuredAgentSessionHistoryRestore: (() => void) | null = null + async getStructuredAgentSessionCreateSupport( worktreeSelector: string, agent: 'claude' | 'codex' diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index 18178c6ef935..70c3f0fdbeae 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -1,8 +1,8 @@ // @ts-nocheck -- mechanically split from OrcaRuntimeService; behavior is covered by AST equivalence and characterization tests. import { randomUUID } from 'node:crypto' import { preserveTerminalRetirementProofs } from './mobile-session-terminal-retirement-proof' -import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import type { TuiAgent } from '../../shared/tui-agent' import type { RuntimeStore } from './runtime-store-contract' @@ -125,9 +125,6 @@ export class OrcaRuntimeWithRuntimeId { protected structuredAgentSessionTabRestorePromise: Promise | null = null - // The history restore a tab restore owes, until a caller that answered with its list starts it. - protected owedStructuredAgentSessionHistoryRestore: (() => void) | null = null - // Whether the last tab restore ran with chats on disk but no host to list them. protected structuredAgentSessionInventoryUnverifiable = false From 7688715afd1a75d43a80776b2027837d9623b4ac Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:37:32 -0700 Subject: [PATCH 018/107] feat(native-chat): store each chat's state with its journal; startup seeds and settles from it Each chat's lifecycle state (whether its next open owes settlement work, and the status summary a session list shows) is stored in journal_session_state, written in the same transaction as the journal row it describes. The open appends one settlement plan built from the same per-entity rules the stored flag reads. Host startup seeds settled listed chats' statuses from stored state and settles every chat that owes work, listed or not, instead of replaying every listed chat after the tab list. --- docs/reference/agent-status-store.md | 14 +- ...aude-subagent-resume-after-restart.test.ts | 13 + .../agent-session-journal/journal-database.ts | 5 +- .../journal-epoch-controller.ts | 6 +- .../journal-epoch-replacement.test.ts | 1 + .../journal-epoch-replacement.ts | 8 +- .../journal-epoch-rollover.ts | 26 +- .../journal-host-database.ts | 5 + .../journal-open-settlement-plan.ts | 107 ++++++ .../journal-pending-submission-recovery.ts | 19 +- .../journal-per-session-import.ts | 3 + .../journal-repair-marker.ts | 3 + .../journal-row-writer.test.ts | 6 +- .../journal-row-writer.ts | 75 +++- .../journal-session-state.ts | 319 ++++++++++++++++++ .../journal-store-collaborators.ts | 104 +++++- .../journal-store-contracts.ts | 2 + .../journal-store-open.ts | 29 -- .../agent-session-journal/journal-store.ts | 50 +-- .../journal-subagent-liveness.test.ts | 28 +- .../journal-subagent-liveness.ts | 30 +- .../agent-session-journal-recovery.ts | 4 +- ...-agent-session-accept-then-deliver.test.ts | 9 +- ...tructured-agent-session-client-delivery.ts | 5 + ...uctured-agent-session-conversation-open.ts | 90 ++--- .../structured-agent-session-conversations.ts | 8 + ...gent-session-dead-generation-settlement.ts | 106 ++---- .../structured-agent-session-host.ts | 6 +- ...tructured-agent-session-open-settlement.ts | 259 ++++++++++++++ .../structured-agent-session-read-restore.ts | 36 +- .../structured-agent-session-reveal.ts | 43 ++- .../structured-agent-session-startup-state.ts | 181 ++++++++++ .../structured-agent-session-status-feed.ts | 160 ++++----- ...structured-agent-session-status-summary.ts | 106 ++++++ src/main/orcad/orcad-entry.ts | 6 + ...structured-agent-session-create-support.ts | 36 +- ...tore-structured-agent-session-tabs-once.ts | 20 +- ...-agent-session-startup-tab-restore.test.ts | 7 +- ...-agent-session-tab-listing-startup.test.ts | 13 +- .../startup/main-process-runtime-launch.ts | 7 + 40 files changed, 1575 insertions(+), 380 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-session-state.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index b7ce4fa16dde..0652d9095438 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -125,11 +125,15 @@ forgot the row would strand a permanently working-looking agent. Two rules the ingest must keep: -- **Never persist a structured row.** The journal is the durable truth for a - structured session and the host republishes on restore. A structured row in - `last-status.json` would hydrate as `restoredUnconfirmed` and then fight the - live republish. The serializer skips rows carrying `structuredHost`, and - hydrate drops any such row found on disk. Applying one therefore also skips +- **Never persist a structured row in the store.** The chat journal database + is the durable truth for a structured session. Each chat's status is stored + there beside its journal (`journal_session_state`), written in the same + transaction as the journal rows it describes and trusted only at the + journal's tip. At host startup the host republishes those rows into the store + without opening the chat. A structured row in `last-status.json` would + hydrate as `restoredUnconfirmed` and fight that republish, so the serializer + still skips rows carrying `structuredHost`, and hydrate still drops any found + on disk. Applying one therefore also skips the persist schedule: the walk and stringify could only reproduce the file that is already on disk, once per debounce window for every streaming chat. - **Never let it fight a hook row.** A structured session has no PTY, so no diff --git a/src/main/claude/claude-subagent-resume-after-restart.test.ts b/src/main/claude/claude-subagent-resume-after-restart.test.ts index ef6c067db9ea..300a977b418a 100644 --- a/src/main/claude/claude-subagent-resume-after-restart.test.ts +++ b/src/main/claude/claude-subagent-resume-after-restart.test.ts @@ -14,6 +14,10 @@ import { import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../native-chat/agent-session-journal/journal-host-database-test-support' import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' +import { + appendOpenSettlement, + planOpenSettlement +} from '../native-chat/agent-session-wire/structured-agent-session-open-settlement' import { createClaudeJournalTranslator } from './claude-structured-journal-translation' import { claudeSubagentGroupBody, claudeSubagentGroupIdentity } from './claude-subagent-group-row' @@ -453,6 +457,15 @@ describe('a Claude subagent resumed after its provider restarted', () => { // contact was lost, not when. await crashed.close() const journal = await openJournal() + // What the conversation's open appends before an acquisition reads the journal. + await appendOpenSettlement( + journal, + planOpenSettlement(journal, null, { acquisition: true, settlesRosters: true }), + 0, + (error) => { + throw error + } + ) const [lost] = rowsListing(journal, 'agent-a')[0]?.agents ?? [] expect(lost).toMatchObject({ id: 'agent-a', state: 'unverifiable' }) expect(lost?.settledAt).toBeUndefined() diff --git a/src/main/native-chat/agent-session-journal/journal-database.ts b/src/main/native-chat/agent-session-journal/journal-database.ts index 2c01a653af38..4fc71f67d95e 100644 --- a/src/main/native-chat/agent-session-journal/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-database.ts @@ -9,6 +9,7 @@ import { hardenSqliteDatabaseFiles } from '../../sqlite/harden-database-files' import { createJournalTablesSql, JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' import { JournalUnreleasedSchemaError } from './journal-open-failure' import { ensureQueuedMessagesTable } from './queued-message-schema' +import { ensureJournalSessionStateTable } from './journal-session-state' export const JOURNAL_BUSY_TIMEOUT_MS = 5000 /** Bounds the WAL a checkpoint leaves behind; SQLite truncates it back to this after a reset. */ @@ -51,8 +52,10 @@ export function openJournalDatabase(dbPath: string): OpenJournalDatabase { createJournalSchema(probe, stored) // Outside `createJournalSchema` on purpose: its early return skips a db // already at the current version, and this table must exist at EVERY - // writable open with no `user_version` bump (see `ensureQueuedMessagesTable`). + // writable open with no `user_version` bump (see `ensureQueuedMessagesTable`). So must each + // chat's stored state. ensureQueuedMessagesTable(probe) + ensureJournalSessionStateTable(probe) hardenSqliteDatabaseFiles(dbPath) transferred = true return { db: probe, readOnly: false } diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index 52a5478c381a..6ce527e9eab4 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -4,7 +4,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import { replaceJournalEpoch, type JournalReplacementItem } from './journal-epoch-replacement' -import { publishNewEpoch } from './journal-epoch-rollover' +import { publishNewEpoch, type JournalEpochStateWriter } from './journal-epoch-rollover' import type { JournalLoad } from './journal-open' import type { AgentJournalEpochReason } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' @@ -22,6 +22,8 @@ export class JournalEpochController { highestFence: () => number cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void + /** The chat's stored state for the new epoch, inside its transaction. */ + writeState: JournalEpochStateWriter } ) {} @@ -33,6 +35,7 @@ export class JournalEpochController { reason, fence, now: this.deps.now(), + writeState: this.deps.writeState, onPublished: this.deps.adopt }) } @@ -69,6 +72,7 @@ export class JournalEpochController { items, now: this.deps.now, mintEpoch: this.deps.mintEpoch, + writeState: this.deps.writeState, onPublished: this.deps.adopt }) return this.deps.cursor() diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts index 646254b4932d..63bccce5582d 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts @@ -69,6 +69,7 @@ function replace(input: { items: input.items, now, mintEpoch: () => `epoch-${clock}`, + writeState: () => undefined, onPublished: input.onPublished ?? (() => undefined) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 8d94c5b2b5b0..1c4acbf7e957 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -1,8 +1,9 @@ // Republishing a live item set into a fresh epoch. // // One transaction: discard the old epoch's rows, insert the epoch row plus the -// replacement items, move the session projection, and retire any repair marker -// — this republished history is exactly what the marker was holding out for. +// replacement items, move the session projection, retire any repair marker +// — this republished history is exactly what the marker was holding out for — +// and store the chat's state for the new epoch. import type { AgentJournalItemBody, @@ -14,6 +15,7 @@ import type { import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' import { clearJournalRepairMarker } from './journal-repair-marker' +import type { JournalEpochStateWriter } from './journal-epoch-rollover' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { buildJournalItemRow, journalRowBase } from './journal-row-builders' import { @@ -42,6 +44,7 @@ export function replaceJournalEpoch(input: { items: readonly JournalReplacementItem[] now: () => number mintEpoch: () => string + writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -82,6 +85,7 @@ export function replaceJournalEpoch(input: { insertJournalRow(db, sessionId, row) } publishJournalSessionEpoch(db, input.identity, epoch) + input.writeState(db, state) }) // COMMIT landed: on disk the superseded rows are gone and this epoch is the diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts index 855f60ac3261..afd6c0e3cb57 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts @@ -1,16 +1,22 @@ // Opening a new epoch. // // One transaction: discard every row of the superseded epoch, insert the new -// epoch row at sequence 1, move the session projection onto it, and retire any -// repair marker the superseded epoch was carrying. Superseded rows are DELETED -// rather than retained — nothing would ever shed them. +// epoch row at sequence 1, move the session projection onto it, retire any +// repair marker the superseded epoch was carrying, and store the chat's state for +// the new epoch. Superseded rows are DELETED rather than retained — nothing would +// ever shed them. import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' import { clearJournalRepairMarker } from './journal-repair-marker' -import { applyJournalRow, createJournalReducerState } from './journal-reducer' +import type Database from '../../sqlite/sync-database' +import { + applyJournalRow, + createJournalReducerState, + type JournalReducerState +} from './journal-reducer' import { deleteJournalEpochRows, insertJournalRow, @@ -19,6 +25,9 @@ import { } from './journal-row-table' import type { AgentJournalEpochReason, JournalRow } from './journal-row-schema' +/** Writes the chat's stored state from the fold an epoch transaction is about to publish. */ +export type JournalEpochStateWriter = (db: Database.Database, state: JournalReducerState) => void + export function publishNewEpoch(input: { database: JournalHostDatabase identity: AgentSessionJournalIdentity @@ -26,6 +35,7 @@ export function publishNewEpoch(input: { reason: AgentJournalEpochReason fence: number now: number + writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -42,6 +52,10 @@ export function publishNewEpoch(input: { } const { sessionId } = input.identity + // Folded first, so the transaction stores the state of the epoch it publishes. + const state = createJournalReducerState(sessionId, input.epoch) + applyJournalRow(state, row) + state.oldestSequence = 1 input.database.transaction((db) => { const retired = readJournalSessionEpoch(db, sessionId) if (retired !== null) { @@ -50,13 +64,11 @@ export function publishNewEpoch(input: { clearJournalRepairMarker(db, sessionId) insertJournalRow(db, sessionId, row) publishJournalSessionEpoch(db, input.identity, input.epoch) + input.writeState(db, state) }) // COMMIT landed: on disk the superseded prefix is gone and this epoch is the // live one. The caller adopts that immediately, or a later failure leaves the // store writing into an epoch that no longer exists. - const state = createJournalReducerState(sessionId, input.epoch) - applyJournalRow(state, row) - state.oldestSequence = 1 input.onPublished({ state, readOnly: false, corrupt: false, malformedRows: 0 }) } diff --git a/src/main/native-chat/agent-session-journal/journal-host-database.ts b/src/main/native-chat/agent-session-journal/journal-host-database.ts index b6999e4ef559..e3f6bb8ee8d2 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database.ts @@ -51,6 +51,11 @@ export class JournalHostDatabase { return this.connection === null } + /** A ROLLBACK failed: the connection may still hold rows that never committed. */ + get isStranded(): boolean { + return this.stranded + } + get db(): Database.Database { const connection = this.connection if (!connection) { diff --git a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts new file mode 100644 index 000000000000..2b83b0a7ab1b --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts @@ -0,0 +1,107 @@ +// What an open owes a chat whose earlier host process is gone, one rule per kind of entity. +// +// The open's settlement plan (`structured-agent-session-open-settlement.ts`) and the stored +// "owes work" flag (`journal-session-state.ts`) both read these same rules, so the flag says a chat +// owes work exactly when its open would write something. Each rule revises its entity out of the +// state that selected it, so once the plan commits the flag reads false again. + +import type { + AgentJournalItemBody, + AgentJournalRenderItem, + AgentJournalSubmission +} from '../../../shared/agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { cancelledJournalPromptBody } from './journal-prompt-body-bounds' +import type { JournalReducerState } from './journal-reducer' +import { staleSubagentRosterRevision } from './journal-subagent-liveness' + +/** A running tool call fails, and a pending approval or question is cancelled. */ +export function openSettlementTerminalBody( + item: Pick +): AgentJournalItemBody | null { + if (item.body.kind === 'tool-call' && item.body.state === 'running') { + return { ...item.body, state: 'failed' } + } + if (item.body.kind === 'approval' || item.body.kind === 'question') { + return item.body.resolution.state === 'pending' ? cancelledJournalPromptBody(item.body) : null + } + return null +} + +/** Every turn record still `running` gets the verdict for its own writer. */ +export function isRunningJournalTurn(item: Pick): boolean { + return readAgentJournalTurn(item.body)?.state === 'running' +} + +/** A handed-over send nothing answered: it becomes a recovered `unknown`. A queued one is not. */ +export function owesRecoveredDispatch(submission: AgentJournalSubmission): boolean { + return ( + !isQueuedAgentJournalSubmission(submission) && + (submission.dispatchState === 'pending' || + (submission.dispatchState === 'unknown' && submission.recovered !== true)) + ) +} + +/** A send accepted and never handed over is a leftover: nothing indexes a chat with one queued. */ +export function isLeftoverQueuedSubmission(submission: AgentJournalSubmission): boolean { + return isQueuedAgentJournalSubmission(submission) +} + +/** The writer fence of an `unverifiable` turn, which only death evidence naming it revises. */ +export function unverifiableTurnOwnerFence( + item: Pick, + itemFence: (itemId: string) => number | undefined +): number | undefined { + return readAgentJournalTurn(item.body)?.state === 'unverifiable' + ? itemFence(item.itemId) + : undefined +} + +export type JournalOwedFacts = { + /** The open writes something whatever the record says. */ + owesWork: boolean + /** Distinct writer fences of `unverifiable` turns, sorted. */ + unverifiableOwnerFences: number[] +} + +/** The plan's rules read without building rows. Rosters are skipped on a corrupt load, as the + * open skips them: that journal still owes a rebuild, and a write would retire the demand. */ +export function owesOpenSettlement( + fold: Pick, + options: { settlesRosters: boolean } +): JournalOwedFacts { + let owesWork = false + for (const submission of fold.submissions.values()) { + if (owesRecoveredDispatch(submission) || isLeftoverQueuedSubmission(submission)) { + owesWork = true + break + } + } + const fences = new Set() + const itemFence = (itemId: string) => fold.itemFences.get(itemId) + for (const item of fold.items.values()) { + const fence = unverifiableTurnOwnerFence(item, itemFence) + if (fence !== undefined) { + fences.add(fence) + } + owesWork ||= + isRunningJournalTurn(item) || + openSettlementTerminalBody(item) !== null || + (options.settlesRosters && staleSubagentRosterRevision(item) !== null) + } + return { owesWork, unverifiableOwnerFences: [...fences].sort((a, b) => a - b) } +} + +/** The record-dependent rule, as the plan applies it: an `unverifiable` turn is revised only by + * death evidence naming its writer. An older build's evidence names no writer, so selects none. */ +export function owesOnOpen( + facts: JournalOwedFacts, + deathEvidence: { ownerFence?: number } | null | undefined +): boolean { + const ownerFence = deathEvidence?.ownerFence + return ( + facts.owesWork || + (ownerFence !== undefined && facts.unverifiableOwnerFences.includes(ownerFence)) + ) +} diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index c252a57d8a57..2a3d892af45b 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -2,6 +2,7 @@ import type { AgentJournalDispatchRejection } from '../../../shared/agent-sessio import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { DISPATCH_DOUBT_HOST_RESTARTED } from './journal-dispatch-doubt-reasons' +import { owesRecoveredDispatch } from './journal-open-settlement-plan' import type { AgentSessionJournal } from './journal-store' /** Settles every submission a process fact left unanswerable. Doubt is never @@ -12,14 +13,7 @@ export async function markJournalPendingSubmissionsUnknown( fence: number, reason: string = DISPATCH_DOUBT_HOST_RESTARTED ): Promise { - const unresolved = journal - .submissions() - .filter( - (entry) => - !isQueuedAgentJournalSubmission(entry) && - (entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true)) - ) + const unresolved = journal.submissions().filter(owesRecoveredDispatch) for (const entry of unresolved) { // An earlier reason already names a sharper fact than "the host restarted". const resolvedReason = @@ -43,14 +37,7 @@ export async function rejectJournalPendingSubmissions( fence: number, rejection: AgentJournalDispatchRejection ): Promise { - const unwritten = journal - .submissions() - .filter( - (entry) => - !isQueuedAgentJournalSubmission(entry) && - (entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true)) - ) + const unwritten = journal.submissions().filter(owesRecoveredDispatch) for (const entry of unwritten) { await journal.resolveDispatch({ clientMessageId: entry.clientMessageId, diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts index 5f85fdded437..0c062ce97e52 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts @@ -19,6 +19,7 @@ // an empty chat founded here would take a new epoch the next open's import could not reconcile. import { createHash } from 'node:crypto' +import { deleteJournalSessionState } from './journal-session-state' import { existsSync } from 'node:fs' import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' @@ -224,6 +225,8 @@ async function copyLegacyJournal( ) } writePerSessionImportMarker(db, sessionId, legacy) + // Whatever was stored described the rows this copy replaced; the next open re-derives it. + deleteJournalSessionState(db, sessionId) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-repair-marker.ts b/src/main/native-chat/agent-session-journal/journal-repair-marker.ts index 785339456712..4a95151b96a8 100644 --- a/src/main/native-chat/agent-session-journal/journal-repair-marker.ts +++ b/src/main/native-chat/agent-session-journal/journal-repair-marker.ts @@ -16,6 +16,7 @@ import type Database from '../../sqlite/sync-database' import type { JournalHostDatabase } from './journal-host-database' import { deleteJournalRowSuffix } from './journal-row-table' +import { deleteJournalSessionState } from './journal-session-state' const SELECT_REPAIR = 'SELECT epoch, content_from FROM journal_repairs WHERE session_id = ?' const UPSERT_REPAIR = `INSERT INTO journal_repairs (session_id, epoch, content_from, repaired_at) @@ -60,6 +61,8 @@ export function deleteJournalRepairedSuffix(input: { return input.database.transaction((db) => { const deleted = deleteJournalRowSuffix(db, input.sessionId, input.epoch, input.fromSeq) db.prepare(UPSERT_REPAIR).run(input.sessionId, input.epoch, input.contentFrom, input.now) + // It described the dropped suffix; the open that follows re-derives it. + deleteJournalSessionState(db, input.sessionId) return deleted }) } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts index aa0078b88a59..8569bdb14f49 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts @@ -68,10 +68,12 @@ describe('journal row writer', () => { readOnly: () => readOnly, highestFence: () => 0, nextSequence: () => sequence, - commit: (committed) => { + apply: (committed) => { committedRows.push(committed) sequence = committed.seq + 1 - } + }, + committed: () => undefined, + recoverFold: () => undefined }) return { writer, committedRows } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index a29585580ebe..eef89bb363fa 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -18,16 +18,25 @@ export type JournalRowWriterDeps = { readOnly: () => boolean highestFence: () => number nextSequence: () => number - commit: (row: JournalRow) => void + /** Folds the row, inside the transaction, so the state row below describes it. */ + apply: (row: JournalRow) => void + /** After COMMIT: observers learn of the row only once it is durable. */ + committed: () => void + /** A transaction that failed after `apply`: the fold is ahead of the disk. Re-folds from disk, + * or closes the chat when the connection is stranded and a re-read would see the row. */ + recoverFold: () => void /** Standing hook run for EVERY appended row — the queued-draft returned * transition rides here so no rejection path can bypass it. Bookkeeping: it * runs in its own savepoint, so its failure is reported and never vetoes the row. */ inTransaction?: JournalRowTransactionHook + /** The chat's stored state, from the fold that now includes the row. Bookkeeping too. */ + writeState?: (db: Database.Database) => void /** After any rollback, so a cache filled inside the transaction cannot outlive it. */ rolledBack?: () => void } const BOOKKEEPING_SAVEPOINT = 'journal_row_bookkeeping' +const STATE_SAVEPOINT = 'journal_session_state' export class JournalRowWriter { constructor(private readonly deps: JournalRowWriterDeps) {} @@ -40,21 +49,26 @@ export class JournalRowWriter { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) const row = build(this.deps.nextSequence(), this.deps.now()) assertJournalFence(row.fence, this.deps.highestFence()) + let applied = false try { // One INSERT: the chat's epoch pointer moves only when the epoch does. this.deps.database().transaction((db) => { insertJournalRow(db, this.deps.sessionId, row) hook?.(db, row) this.runBookkeeping(db, row) + applied = true + this.deps.apply(row) + this.runStateWrite(db, row) }) } catch (error) { this.deps.rolledBack?.() + if (applied) { + this.deps.recoverFold() + } throw error } - // COMMIT landed, so the row is durable: adopt it before anything that can - // fail. Rejecting here instead would leave the next append reusing a - // sequence the table already holds. - this.deps.commit(row) + // COMMIT landed and the fold already holds the row; only now do observers hear of it. + this.deps.committed() return row }) } @@ -73,17 +87,50 @@ export class JournalRowWriter { if (!hook) { return } - db.exec(`SAVEPOINT ${BOOKKEEPING_SAVEPOINT}`) + this.inSavepoint( + db, + BOOKKEEPING_SAVEPOINT, + row, + () => hook(db, row), + () => { + this.deps.rolledBack?.() + // The draft store re-derives what this missed from the committed rows: at open, and in + // the drain step before a draft sends. + return 'row bookkeeping skipped' + } + ) + } + + private runStateWrite(db: Database.Database, row: JournalRow): void { + const write = this.deps.writeState + if (write) { + // Left behind the tip on failure, so the next open or startup re-derives it. + this.inSavepoint( + db, + STATE_SAVEPOINT, + row, + () => write(db), + () => 'state row skipped' + ) + } + } + + /** Bookkeeping never vetoes the row: its failure rolls back to the savepoint and is reported. */ + private inSavepoint( + db: Database.Database, + name: string, + row: JournalRow, + run: () => void, + onFailure: () => string + ): void { + db.exec(`SAVEPOINT ${name}`) try { - hook(db, row) - db.exec(`RELEASE ${BOOKKEEPING_SAVEPOINT}`) + run() + db.exec(`RELEASE ${name}`) } catch (error) { - db.exec(`ROLLBACK TO ${BOOKKEEPING_SAVEPOINT}`) - db.exec(`RELEASE ${BOOKKEEPING_SAVEPOINT}`) - this.deps.rolledBack?.() - // The draft store re-derives what this missed from the committed rows: at open, and in - // the drain step before a draft sends. - console.warn('[journal-append] row bookkeeping skipped:', { + db.exec(`ROLLBACK TO ${name}`) + db.exec(`RELEASE ${name}`) + console.warn(`[journal-append] ${onFailure()}:`, { sessionId: this.deps.sessionId, kind: row.kind, error: error instanceof Error ? error.message : String(error) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts new file mode 100644 index 000000000000..9964acb4d132 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -0,0 +1,319 @@ +// Each chat's lifecycle state, stored beside its journal. +// +// One row per chat: whether its next open owes settlement work (see +// journal-open-settlement-plan.ts), and the status summary a session list shows. Written in the +// SAME transaction as the journal row it describes, and trusted only while its `(epoch, seq)` is the +// chat's live tip, so a write this build missed (an older build's append, a failed savepoint) +// reads as stale and the chat is re-derived. Deleting a row is always safe for the same reason. +// +// Created at every writable open with no `user_version` bump, as the draft table is +// (queued-message-schema.ts): an older build ignores it and stays writable. + +import type Database from '../../sqlite/sync-database' +import { isAgentJournalTurnOutcome } from '../../../shared/agent-turn-outcome' +import { + projectStructuredAgentSessionStatusState, + type StructuredAgentSessionStatusProjection +} from '../../../shared/structured-agent-session-projection' +import { owesOpenSettlement, type JournalOwedFacts } from './journal-open-settlement-plan' +import { renderJournalState, type JournalReducerState } from './journal-reducer' + +/** Version of the rules that derive a row. A row of another version reads as absent. Bump it with + * any change to what `deriveJournalSessionState` produces; the golden test pins it. */ +export const JOURNAL_SESSION_STATE_VERSION = 1 + +export function ensureJournalSessionStateTable(db: Database.Database): void { + db.exec(` +CREATE TABLE IF NOT EXISTS journal_session_state ( + session_id TEXT PRIMARY KEY, + state_version INTEGER NOT NULL, + epoch TEXT NOT NULL, + seq INTEGER NOT NULL, + owes_work INTEGER NOT NULL, + unverifiable_owner_fences TEXT, + summary_json TEXT, + last_activity_at INTEGER NOT NULL, + written_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS journal_session_state_owed + ON journal_session_state (session_id) + WHERE owes_work = 1 OR unverifiable_owner_fences IS NOT NULL; +`) +} + +export type DerivedJournalSessionState = JournalOwedFacts & { + epoch: string + seq: number + /** Only while nothing is owed: an owing chat is settled and republished by its open. */ + summary: StructuredAgentSessionStatusProjection | null + lastActivityAt: number +} + +export type StoredJournalSessionState = DerivedJournalSessionState & { + stateVersion: number + writtenAt: number +} + +export type JournalSessionStateInput = { + /** False while the chat's load is corrupt: its open settles no roster either. */ + settlesRosters: boolean + currentFence?: number +} + +export function deriveJournalSessionState( + state: JournalReducerState, + input: JournalSessionStateInput +): DerivedJournalSessionState { + const facts = owesOpenSettlement(state, input) + let summary: StructuredAgentSessionStatusProjection | null = null + if (!facts.owesWork) { + // Fence-independent here: nothing unanswered or queued is left for the fence to judge. + const snapshot = renderJournalState(state) + summary = projectStructuredAgentSessionStatusState( + snapshot.items, + snapshot.submissions, + input.currentFence + ).summary + } + return { + ...facts, + epoch: state.epoch, + seq: state.lastSequence, + summary, + lastActivityAt: state.lastActivityAt + } +} + +const UPSERT_STATE = `INSERT INTO journal_session_state (session_id, state_version, epoch, seq, + owes_work, unverifiable_owner_fences, summary_json, last_activity_at, written_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(session_id) DO UPDATE SET + state_version = excluded.state_version, epoch = excluded.epoch, seq = excluded.seq, + owes_work = excluded.owes_work, unverifiable_owner_fences = excluded.unverifiable_owner_fences, + summary_json = excluded.summary_json, last_activity_at = excluded.last_activity_at, + written_at = excluded.written_at` +const DELETE_STATE = 'DELETE FROM journal_session_state WHERE session_id = ?' +const STATE_COLUMNS = `st.state_version AS state_version, st.epoch AS state_epoch, st.seq AS seq, + st.owes_work AS owes_work, st.unverifiable_owner_fences AS unverifiable_owner_fences, + st.summary_json AS summary_json, st.last_activity_at AS last_activity_at, + st.written_at AS written_at` +const SELECT_STATE = `SELECT ${STATE_COLUMNS} FROM journal_session_state st WHERE st.session_id = ?` + +export function writeJournalSessionState( + db: Database.Database, + sessionId: string, + derived: DerivedJournalSessionState, + writtenAt: number +): void { + db.prepare(UPSERT_STATE).run( + sessionId, + JOURNAL_SESSION_STATE_VERSION, + derived.epoch, + derived.seq, + derived.owesWork ? 1 : 0, + derived.unverifiableOwnerFences.length > 0 + ? JSON.stringify(derived.unverifiableOwnerFences) + : null, + derived.summary ? JSON.stringify(derived.summary) : null, + derived.lastActivityAt, + writtenAt + ) +} + +export function deleteJournalSessionState(db: Database.Database, sessionId: string): void { + db.prepare(DELETE_STATE).run(sessionId) +} + +/** This build's row for the chat, or null when there is none or another version wrote it. */ +export function readJournalSessionState( + db: Database.Database, + sessionId: string +): StoredJournalSessionState | null { + const row = db.prepare(SELECT_STATE).get(sessionId) + return row ? parseStoredState(row) : null +} + +/** A chat's stored state beside its live tip, for the startup seed. */ +export type JournalSessionStateAtTip = { + sessionId: string + /** Null when absent, of another version, behind the tip, or older than a repair. */ + current: StoredJournalSessionState | null +} + +// Bounded well under SQLite's host-parameter limit. +const IN_LIST_CHUNK = 500 + +/** One query per chunk of ids; ids with no journal yet are not returned. */ +export function readJournalSessionStatesAtTip( + db: Database.Database, + sessionIds: readonly string[] +): JournalSessionStateAtTip[] { + const results: JournalSessionStateAtTip[] = [] + for (let start = 0; start < sessionIds.length; start += IN_LIST_CHUNK) { + const chunk = sessionIds.slice(start, start + IN_LIST_CHUNK) + const rows = db + .prepare( + `SELECT s.session_id AS session_id, s.epoch AS live_epoch, + (SELECT MAX(r.seq) FROM journal_rows r + WHERE r.session_id = s.session_id AND r.epoch = s.epoch) AS tip, + (SELECT p.repaired_at FROM journal_repairs p + WHERE p.session_id = s.session_id AND p.epoch = s.epoch) AS repaired_at, + ${STATE_COLUMNS} +FROM journal_sessions s LEFT JOIN journal_session_state st ON st.session_id = s.session_id +WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` + ) + .all(...chunk) + for (const row of rows) { + if (typeof row.session_id !== 'string') { + continue + } + const stored = parseStoredState(row) + const atTip = + stored !== null && + stored.epoch === row.live_epoch && + stored.seq === row.tip && + (typeof row.repaired_at !== 'number' || row.repaired_at <= stored.writtenAt) + results.push({ sessionId: row.session_id, current: atTip ? stored : null }) + } + } + return results +} + +/** Every chat this build's rows say may owe work at open, through the partial index. */ +export function readOwedJournalSessionStates( + db: Database.Database +): (JournalOwedFacts & { sessionId: string })[] { + return db + .prepare( + `SELECT session_id, owes_work, unverifiable_owner_fences FROM journal_session_state +WHERE (owes_work = 1 OR unverifiable_owner_fences IS NOT NULL) AND state_version = ?` + ) + .all(JOURNAL_SESSION_STATE_VERSION) + .flatMap((row) => + typeof row.session_id === 'string' + ? [ + { + sessionId: row.session_id, + owesWork: row.owes_work === 1, + unverifiableOwnerFences: parseFences(row.unverifiable_owner_fences) + } + ] + : [] + ) +} + +/** Chats with a draft the drain would send on open: waiting, not held, and the queue unpaused. */ +export function readSessionsWithDrainableDrafts(db: Database.Database): Set { + const rows = db + .prepare( + `SELECT DISTINCT q.session_id AS session_id FROM queued_messages q +WHERE q.state = 'waiting' AND q.hold_reason IS NULL + AND NOT EXISTS (SELECT 1 FROM queued_message_pauses p WHERE p.session_id = q.session_id)` + ) + .all() + return new Set( + rows.flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) + ) +} + +function parseStoredState(row: Record): StoredJournalSessionState | null { + if ( + row.state_version !== JOURNAL_SESSION_STATE_VERSION || + typeof row.state_epoch !== 'string' || + typeof row.seq !== 'number' || + typeof row.last_activity_at !== 'number' || + typeof row.written_at !== 'number' + ) { + return null + } + const owesWork = row.owes_work === 1 + const summary = owesWork ? null : parseSummary(row.summary_json) + if (!owesWork && !summary) { + return null + } + return { + stateVersion: JOURNAL_SESSION_STATE_VERSION, + epoch: row.state_epoch, + seq: row.seq, + owesWork, + unverifiableOwnerFences: parseFences(row.unverifiable_owner_fences), + summary, + lastActivityAt: row.last_activity_at, + writtenAt: row.written_at + } +} + +function parseJson(value: unknown): unknown { + if (typeof value !== 'string') { + return null + } + try { + return JSON.parse(value) + } catch { + return null + } +} + +function parseFences(value: unknown): number[] { + const parsed = parseJson(value) + return Array.isArray(parsed) + ? parsed.filter((fence): fence is number => Number.isSafeInteger(fence)) + : [] +} + +const STATUSES = ['idle', 'working', 'attention'] as const + +function parseStatus(value: unknown): StructuredAgentSessionStatusProjection['status'] | undefined { + return value === null ? null : STATUSES.find((status) => status === value) +} + +/** Known keys only: a newer writer's extra keys are ignored rather than trusted. */ +function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | null { + const parsed = parseJson(value) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + return null + } + const source = new Map(Object.entries(parsed)) + const status = parseStatus(source.get('status')) + const latestPrompt = source.get('latestPrompt') + if (status === undefined || typeof latestPrompt !== 'string') { + return null + } + const text = (key: string): string | undefined => { + const field = source.get(key) + return typeof field === 'string' ? field : undefined + } + const turnOutcome = source.get('turnOutcome') + const statusStartedAt = source.get('statusStartedAt') + const toolName = text('toolName') + const toolInput = text('toolInput') + const lastAssistantMessage = text('lastAssistantMessage') + return { + status, + latestPrompt, + ...(toolName !== undefined ? { toolName } : {}), + ...(toolInput !== undefined ? { toolInput } : {}), + ...(lastAssistantMessage !== undefined ? { lastAssistantMessage } : {}), + ...(isAgentJournalTurnOutcome(turnOutcome) ? { turnOutcome } : {}), + ...(typeof statusStartedAt === 'number' ? { statusStartedAt } : {}) + } +} + +/** + * The open's re-derive: rewrites the chat's row when it is absent, another version's, off the + * fold's tip, or older than a repair. What an older build, an import, a repair or a failed + * savepoint left behind is caught here. Bookkeeping: the caller logs a failure and goes on. + */ +export function ensureJournalSessionStateCurrent( + db: Database.Database, + state: JournalReducerState, + input: JournalSessionStateInput, + now: number +): boolean { + const [stored] = readJournalSessionStatesAtTip(db, [state.sessionId]) + if (stored?.current?.epoch === state.epoch && stored.current.seq === state.lastSequence) { + return false + } + writeJournalSessionState(db, state.sessionId, deriveJournalSessionState(state, input), now) + return true +} diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 16d3795cc6ab..6088467d614b 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -18,6 +18,16 @@ import { JournalRowWriter } from './journal-row-writer' import { restoreJournalStore } from './journal-store-restore' import type { JournalRow } from './journal-row-schema' import type { AgentSessionJournal } from './journal-store' +import type Database from '../../sqlite/sync-database' +import { applyJournalRow } from './journal-reducer' +import { readJournalRowsAfterCursor, replayJournal } from './journal-open' +import { readJournalSince } from './journal-cursor' +import type { JournalReadSince } from './journal-store-contracts' +import { + deriveJournalSessionState, + ensureJournalSessionStateCurrent, + writeJournalSessionState +} from './journal-session-state' export type JournalStoreHost = { /** Fires the journal's commit listener for a durable change that appended no @@ -40,7 +50,16 @@ export type JournalStoreHost = { setReadOnly: (readOnly: boolean) => void cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void - commit: (row: JournalRow) => void + /** Replaces the fold alone, quietly: a failed append's re-fold from disk. */ + replaceState: (state: JournalReducerState) => void + /** The chat can no longer trust its fold or its connection: refuses every later write. */ + strand: () => void + /** Whether this open's replay found an unusable prefix. */ + openedCorrupt: () => boolean + /** The conversation's fence, which the stored status reads as the status feed does. */ + currentFence: () => number | undefined + /** A per-chat file's copy is still owed: the fold is not the database's yet. */ + importPending: () => boolean /** Records whether the open's replay found an unusable prefix. */ setOpenedCorrupt: (corrupt: boolean) => void malformedRows: () => number @@ -58,9 +77,18 @@ export type JournalStoreCollaborators = { /** Restores the store's state from disk. Owned here because it needs the same * collaborators the constructor just built. */ restore: () => Promise + ensureSessionState: () => void + readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince } export function createJournalStoreCollaborators(host: JournalStoreHost): JournalStoreCollaborators { + const writeState = (db: Database.Database, state: JournalReducerState) => + writeJournalSessionState( + db, + host.identity.sessionId, + deriveJournalSessionState(state, journalSessionStateInput(host)), + host.now() + ) const epochController = new JournalEpochController({ identity: host.identity, now: host.now, @@ -71,7 +99,8 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal setReadOnly: host.setReadOnly, highestFence: () => host.state().highestFence, cursor: host.cursor, - adopt: host.adopt + adopt: host.adopt, + writeState }) const queuedMessages = new JournalQueuedMessages({ sessionId: host.identity.sessionId, @@ -85,6 +114,24 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal return { epochController, queuedMessages, + ensureSessionState: () => ensureStoredSessionState(host), + readSince: (cursor, limit) => + readJournalSince( + { + state: host.state(), + rowsAfter: (afterSequence) => + readJournalRowsAfterCursor( + host.database().db, + host.identity.sessionId, + host.state().epoch, + afterSequence, + limit + ), + readOnly: host.readOnly() + }, + cursor, + host.cursor + ), // Behind the stored fact: settles drafts whose consumed submission the loaded journal shows // refused (a downgrade wrote no hook), then prunes. Bookkeeping, never failing the open. restore: () => @@ -99,7 +146,10 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal readOnly: host.readOnly, highestFence: () => host.state().highestFence, nextSequence: () => host.state().lastSequence + 1, - commit: host.commit, + apply: (row) => applyJournalRow(host.state(), row), + committed: host.notifyCommitted, + recoverFold: () => recoverJournalFold(host), + writeState: (db) => writeState(db, host.state()), // Every rejection is a dispatch row through this one writer; the draft // returned-transition rides it so no path can bypass the hook. inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), @@ -116,3 +166,51 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal }) } } + +function journalSessionStateInput(host: Pick) { + return { settlesRosters: !host.openedCorrupt(), currentFence: host.currentFence() } +} + +function ensureStoredSessionState(host: JournalStoreHost): void { + const database = host.database() + if (host.readOnly() || database.readOnly || host.importPending()) { + return + } + try { + ensureJournalSessionStateCurrent( + database.db, + host.state(), + journalSessionStateInput(host), + host.now() + ) + } catch (error) { + console.warn('[agent-session-journal] re-deriving the stored chat state failed', { + sessionId: host.identity.sessionId, + error + }) + } +} + +/** + * An append whose transaction failed after its row was folded: the fold is ahead of the disk, and + * an in-place apply cannot be undone, so the chat is folded again from what committed. A stranded + * connection may still hold the uncommitted row, so a re-read would adopt it: the chat closes + * instead, and its next open reads the disk once the host database's own rollback goes through. + */ +function recoverJournalFold( + host: Pick +): void { + const database = host.database() + if (!database.isStranded) { + try { + const reloaded = replayJournal(database.db, host.identity.sessionId) + if (reloaded) { + host.replaceState(reloaded.state) + return + } + } catch (error) { + console.warn('[agent-session-journal] re-folding after a failed append failed', error) + } + } + host.strand() +} diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 29f0db6bed63..3058768e7ec9 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -21,6 +21,8 @@ export type AgentSessionJournalOptions = { mintEpoch?: () => string /** A restore's open: see `AgentSessionJournal.whenImported`. */ deferPerSessionImport?: boolean + /** The conversation's fence, which the chat's stored status reads as the status feed does. */ + currentFence?: () => number | undefined } export type JournalReadSince = diff --git a/src/main/native-chat/agent-session-journal/journal-store-open.ts b/src/main/native-chat/agent-session-journal/journal-store-open.ts index a7163860be96..7a30e4355ce7 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-open.ts @@ -9,7 +9,6 @@ import { } from './journal-file-format-remnant' import type { JournalLoad } from './journal-open' import { journalRepairDisclosure, type JournalRepairDisclosure } from './journal-repair-disclosure' -import { staleSubagentRosterRevisions } from './journal-subagent-liveness' /** What any of this file's disclosures hands the store — a repair's, or the * pre-SQLite notice's. Same shape, and neither is only a repair. */ @@ -69,7 +68,6 @@ export async function openJournalStoreState(input: { const disclosure = journalRepairDisclosure({ malformedRows: input.malformedRows() }) await input.appendItem(disclosure.identity, disclosure.body, input.highestFence()) } - await settleStaleSubagentRosters(input, loaded) // Founding the epoch and appending the row are two transactions, and a // committed epoch sends every later open down this branch instead. Anything // that interrupts between them — a quit during startup restore, a failed @@ -110,30 +108,3 @@ async function discloseFileFormatRemnant(input: { const disclosure = journalFileFormatRemnantDisclosure({ transcriptPath, agent: input.agent }) await input.appendItem(disclosure.identity, disclosure.body, input.highestFence()) } - -/** - * Retires a `working` subagent roster the previous host never got to settle. - * - * Skipped on a corrupt load: that journal is still owed a rebuild from provider - * history, and content written past the repair's free sequence retires the - * demand for it. - */ -async function settleStaleSubagentRosters( - input: { - appendItem: ( - identity: AgentJournalItemIdentity, - body: AgentJournalItemBody, - fence: number - ) => Promise - highestFence: () => number - readOnly: () => boolean - }, - loaded: JournalLoad -): Promise { - if (input.readOnly() || loaded.corrupt) { - return - } - for (const revision of staleSubagentRosterRevisions(loaded.state.items.values())) { - await input.appendItem(revision.identity, revision.body, input.highestFence()) - } -} diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 5b046ab421f2..77226aaed625 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -25,16 +25,14 @@ import { newestStructuredAgentSessionTurnBySequence } from '../../../shared/structured-agent-session-live-turn' import type { JournalReplacementItem } from './journal-epoch-replacement' -import { readJournalSince } from './journal-cursor' import type { JournalHostDatabase } from './journal-host-database' -import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open' +import type { JournalLoad } from './journal-open' import { markJournalPendingSubmissionsUnknown, rejectJournalPendingSubmissions, rejectJournalQueuedSubmissions } from './journal-pending-submission-recovery' import { - applyJournalRow, createJournalReducerState, renderJournalState, resolveJournalItemId, @@ -81,12 +79,17 @@ export class AgentSessionJournal { private openedCorrupt = false private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } private onCommitted: (() => void) | null = null + private onStranded: (() => void) | null = null private readonly queue: JournalWriteQueue private readonly rowWriter: JournalRowWriter private readonly epochController: JournalEpochController private readonly itemAppender: JournalItemAppender private readonly lifecycleBatchAppender: JournalLifecycleBatchAppender private readonly restore: () => Promise + /** Rewrites the chat's stored state if it does not describe this fold. Bookkeeping: never + * fails the open that calls it. */ + readonly ensureSessionState: () => void + readonly readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince /** Draft rows queued while the agent works; never reducer input or owed work. */ readonly queuedMessages: JournalQueuedMessages @@ -117,10 +120,16 @@ export class AgentSessionJournal { this.adoptLoadedJournal(loaded) this.onCommitted?.() }, - commit: (row) => { - applyJournalRow(this.state, row) - this.onCommitted?.() + replaceState: (state) => { + this.state = state + }, + strand: () => { + this.queue.markClosed() + this.onStranded?.() }, + openedCorrupt: () => this.openedCorrupt, + currentFence: options.currentFence ?? (() => undefined), + importPending: () => this.queue.owing, setOpenedCorrupt: (corrupt) => { this.openedCorrupt = corrupt }, @@ -138,6 +147,8 @@ export class AgentSessionJournal { this.lifecycleBatchAppender = collaborators.lifecycleBatchAppender this.queuedMessages = collaborators.queuedMessages this.restore = collaborators.restore + this.ensureSessionState = collaborators.ensureSessionState + this.readSince = collaborators.readSince } get isReadOnly(): boolean { @@ -186,6 +197,11 @@ export class AgentSessionJournal { this.onCommitted = listener } + /** Told once this handle closed itself because its connection was stranded mid-append. */ + observeStranded(listener: () => void): void { + this.onStranded = listener + } + /** * Resolves once the chat's rows are in the host's database. A restore's open serves a chat still * in its per-chat file from a read-only fold of it; the copy runs before the chat's first write, @@ -251,6 +267,9 @@ export class AgentSessionJournal { /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt + /** The highest fence any row carries: a writer that revises what an earlier one left. */ + highestFence = (): number => this.state.highestFence + /** Fence of the writer that created the item, while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) @@ -268,25 +287,6 @@ export class AgentSessionJournal { canonicalItemId = (itemId: string): string => resolveJournalItemId(this.state, itemId) - readSince(cursor: AgentJournalCursor, limit?: number): JournalReadSince { - return readJournalSince( - { - state: this.state, - rowsAfter: (afterSequence) => - readJournalRowsAfterCursor( - this.database.db, - this.identity.sessionId, - this.state.epoch, - afterSequence, - limit - ), - readOnly: this.readOnly - }, - cursor, - () => this.cursor() - ) - } - /** Upsert by stable identity. The revision is assigned here so a caller * cannot accidentally publish a revision the reducer will drop. */ appendItem( diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts index 6b519ce35cd8..5f751eeef7d3 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts @@ -21,6 +21,10 @@ import { import type { openAgentSessionJournal } from './journal-store-factory' import { createTrackedJournalOpener } from './journal-host-database-test-support' import { staleSubagentRosterRevisions } from './journal-subagent-liveness' +import { + appendOpenSettlement, + planOpenSettlement +} from '../agent-session-wire/structured-agent-session-open-settlement' const IDENTITY: AgentSessionJournalIdentity = { sessionId: 'session-1', @@ -52,6 +56,20 @@ async function open(overrides: Partial { + throw error + } + ) + return journal +} + /** The row as the producer writes it: the structured block plus its twin. */ function rosterRow(agents: NativeChatSubagentEntry[]) { return { @@ -236,7 +254,7 @@ describe('journal reopen after the writing host is gone', () => { // The host dies without ever settling them — no `ended`, so no session sweep. await live.close() - const reopened = await open() + const reopened = await reopenAfterHostGone() const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(rosterOf(afterRestart.body)).toMatchObject([ @@ -256,7 +274,7 @@ describe('journal reopen after the writing host is gone', () => { const before = live.snapshot().items.length await live.close() - const reopened = await open() + const reopened = await reopenAfterHostGone() expect(reopened.snapshot().items).toHaveLength(before) expect(reopened.snapshot().items.at(-1)?.revision).toBe(2) }) @@ -273,7 +291,7 @@ describe('journal reopen after the writing host is gone', () => { expect(twinOf(beforeRestart.body)).toBe('Started background command "sleep 20"') await live.close() - const reopened = await open() + const reopened = await reopenAfterHostGone() const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(taskOf(afterRestart.body)).toMatchObject({ state: 'unverifiable' }) @@ -289,11 +307,11 @@ describe('journal reopen after the writing host is gone', () => { }) await live.close() - const once = await open() + const once = await reopenAfterHostGone() const revision = once.snapshot().items.at(-1)?.revision await once.close() - const twice = await open() + const twice = await reopenAfterHostGone() expect(twice.snapshot().items.at(-1)?.revision).toBe(revision) }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts index ec81136688e3..d685724a098e 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts @@ -52,21 +52,31 @@ export function staleSubagentRosterRevisions( ): JournalSubagentLivenessRevision[] { const revisions: JournalSubagentLivenessRevision[] = [] for (const item of items) { - const body = item.body - if (body.kind !== 'message' || !body.blocks.some(hasStaleLiveWork)) { - continue + const revision = staleSubagentRosterRevision(item) + if (revision) { + revisions.push(revision) } - // A key that will not parse cannot be re-addressed, and appending under a - // fresh identity would duplicate the row rather than revise it. - const identity = parseAgentJournalItemKey(item.itemId) - if (!identity || agentJournalItemKey(identity) !== item.itemId) { - continue - } - revisions.push({ identity, body: { ...body, blocks: settleBlocks(body.blocks) } }) } return revisions } +/** One row's revision, or null when it claims no live child. */ +export function staleSubagentRosterRevision( + item: Pick +): JournalSubagentLivenessRevision | null { + const body = item.body + if (body.kind !== 'message' || !body.blocks.some(hasStaleLiveWork)) { + return null + } + // A key that will not parse cannot be re-addressed, and appending under a + // fresh identity would duplicate the row rather than revise it. + const identity = parseAgentJournalItemKey(item.itemId) + if (!identity || agentJournalItemKey(identity) !== item.itemId) { + return null + } + return { identity, body: { ...body, blocks: settleBlocks(body.blocks) } } +} + function hasStaleLiveWork(block: NativeChatBlock): boolean { return hasWorkingChild(block) || hasLiveBackgroundTask(block) } diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts index 42da9a986dad..68e055141c6c 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts @@ -48,11 +48,13 @@ export async function openAgentSessionJournalWithRecovery(input: { /** Resolve directly to a transcript instead of discovering it by session id. */ historyFilePath?: string | null deferPerSessionImport?: boolean + currentFence?: () => number | undefined }): Promise { const journal = await openAgentSessionJournal({ identity: input.identity, database: input.database, - deferPerSessionImport: input.deferPerSessionImport + deferPerSessionImport: input.deferPerSessionImport, + currentFence: input.currentFence }) if (!journal.needsRebuild) { return { journal, recovery: null } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts index b7371ee40f36..04f8d50e4fb6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -643,17 +643,20 @@ describe('a start whose failure the delivery loop settles before the exit is pub }) describe('Stop withdraws what is queued', () => { - it('withdraws a crash leftover ahead of any delivery step (W17a)', async () => { + it('finds a crash leftover already settled by the open Stop reaches it through (W17a)', async () => { await writeAsEarlierProcess(async (journal, fence) => { await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) }) - // Stop's own open wakes the delivery loop, whose first step queues behind this Stop. + // Stop's own open appends the chat's settlement plan, which rejects what a gone process left + // queued before Stop acts: nothing is handed over, and the reason names the restart. expect(await stop()).toMatchObject({ ok: true }) expect(await submission('leftover')).toMatchObject({ dispatchState: 'rejected', - reason: DISPATCH_REJECTED_CANCELLED + reason: agentSessionFailureWords(agentSessionFailureFact('hostRestarted'), { + surface: 'rejection' + }).reason }) expect(acquire).toHaveBeenCalledTimes(1) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index fee7f9e1d3d3..b708deb45b1e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -11,6 +11,7 @@ import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation- import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' import { createStructuredAgentSessionHostStatusFeed, + type StructuredAgentSessionStatusFeed, type StructuredAgentSessionStatusSubscriber } from './structured-agent-session-status-feed' import { @@ -72,6 +73,10 @@ export class StructuredAgentSessionClientDelivery { publishRestored = (sessionId: string): void => this.statusFeed.publish(sessionId, undefined, { replay: true }) + /** A row from the state stored beside a journal nobody has opened. */ + seedStatus: StructuredAgentSessionStatusFeed['seed'] = (record, stored) => + this.statusFeed.seed(record, stored) + subscribeStatus = (subscriber: StructuredAgentSessionStatusSubscriber): (() => void) => this.statusFeed.subscribe(subscriber) forgetStatus = (sessionId: string): void => this.statusFeed.forget(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index ec938f17f13b..54420122f1e9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -1,16 +1,16 @@ // The one way a conversation's journal becomes open on this host: for a send, for a reader, and // for an attach that finds none open. // -// It opens with recovery, so an unusable journal is rebuilt rather than refused, and it marks what -// an earlier host process handed over and left unanswered as in doubt, and settles what it left -// running — the crash boundary. That -// needs no lease: provider history decides such a row later, under a won lease, in the attach. A -// row an earlier process accepted and never handed over is the delivery loop's, which the open -// wakes. Nothing here starts a provider child. +// It opens with recovery, so an unusable journal is rebuilt rather than refused, and it appends +// the chat's open settlement plan (structured-agent-session-open-settlement.ts): what an earlier +// host process handed over and left unanswered becomes doubt, what it accepted and never handed +// over is rejected, and what it left running is settled — the crash boundary. That needs no lease: +// provider history decides a doubtful row later, under a won lease, in the attach. Nothing here +// starts a provider child. The plan's appends keep the chat's stored state current, and the open +// re-derives it when something else (an older build, an import, a repair) left it behind. import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from '../agent-session-journal/journal-host-database' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecord } from '../../../shared/agent-session-record' @@ -21,7 +21,12 @@ import { } from './structured-agent-session-attach' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { + appendGoneGenerationSettlement, + appendOpenSettlement, + planOpenSettlement, + type OpenSettlementRecordFacts +} from './structured-agent-session-open-settlement' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionHostDeps, @@ -81,7 +86,8 @@ export async function openStructuredAgentSessionConversation( /** The open itself, indexed by nobody yet: the caller adopts the result. */ export async function openStructuredAgentSessionConversationJournal( - deps: Omit, + deps: Omit & + Partial>, record: AgentSessionRecord, options: StructuredAgentSessionConversationOpenOptions = {} ): Promise { @@ -97,21 +103,21 @@ export async function openStructuredAgentSessionConversationJournal( database: deps.journalDatabase, fence, historyFilePath: (await deps.adapter.historyFilePath?.({ identity })) ?? null, - deferPerSessionImport: options.deferPerSessionImport + deferPerSessionImport: options.deferPerSessionImport, + // The fence the status feed reads, which a child's end moves after this open. + currentFence: () => deps.store?.getRecord(sessionId)?.lease.runtimeFence ?? fence }) - try { - // A queued row found here is a leftover the delivery loop's first step rejects; a handed-over - // one is only doubt, which provider history decides under a won lease. - await opened.journal.markPendingSubmissionsUnknown(fence) - } catch (error) { - deps.onEventSinkError?.({ sessionId, error }) - } // No child in this process writes to a journal nobody had open, so whatever it shows running // belongs to a generation that is gone, whatever the lease still claims. Settled before any - // reader or child sees it. - if (!options.acquisition) { - await settleGoneGeneration(deps, record, opened.journal) - } + // reader or child sees it; an acquisition settles against the generation it takes instead. + const plan = planOpenSettlement(opened.journal, openSettlementRecordFacts(record), { + acquisition: options.acquisition, + settlesRosters: !opened.journal.needsRebuild + }) + await appendOpenSettlement(opened.journal, plan, fence, (error) => + deps.onEventSinkError?.({ sessionId, error }) + ) + opened.journal.ensureSessionState() return { session: { journal: opened.journal, params, child: null }, reset: opened.recovery?.reset ?? null @@ -129,28 +135,34 @@ export async function resettleOpenStructuredAgentSessionConversation( session: StructuredAgentSessionHostSession | undefined ): Promise { const record = deps.store.getRecord(sessionId) - if (session && record?.lease.deathEvidence) { - await settleGoneGeneration(deps, record, session.journal) + if (!session || !record?.lease.deathEvidence) { + return } -} - -async function settleGoneGeneration( - deps: Pick, - record: AgentSessionRecord, - journal: AgentSessionJournal -): Promise { + const { goneGeneration } = planOpenSettlement( + session.journal, + openSettlementRecordFacts(record), + { settlesRosters: false } + ) try { - await settleStaleStructuredAgentSessionState({ - journal, - sessionId: record.sessionId, - fence: record.lease.runtimeFence, - acquisitionGeneration: null, - deathEvidence: record.lease.deathEvidence ?? null, - failureTextContext: structuredAgentSessionFailureWordsContext(record) - }) + if (goneGeneration && goneGeneration.mutations.length > 0) { + await appendGoneGenerationSettlement( + session.journal, + goneGeneration, + record.lease.runtimeFence + ) + } } catch (error) { // Best effort: the next open or acquire re-derives it. - deps.onEventSinkError?.({ sessionId: record.sessionId, error }) + deps.onEventSinkError?.({ sessionId, error }) + } +} + +export function openSettlementRecordFacts(record: AgentSessionRecord): OpenSettlementRecordFacts { + return { + sessionId: record.sessionId, + fence: record.lease.runtimeFence, + deathEvidence: record.lease.deathEvidence ?? null, + failureTextContext: structuredAgentSessionFailureWordsContext(record) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts index c312d6013fd5..085f63b32c7c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts @@ -50,6 +50,14 @@ export class StructuredAgentSessionConversations extends Map< } }) }) + // A handle that closed itself mid-append no longer matches the disk: the next use reopens it. + // One with a child is left to that child's own sink-failure recovery. + journal.observeStranded?.(() => { + const current = this.get(sessionId) + if (current?.journal === journal && !current.child) { + this.delete(sessionId) + } + }) this.activity.set(sessionId, this.delivery.now()) const adopted = super.set(sessionId, session) this.delivery.onOpened?.(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 72caac40844d..fe8a0aecd221 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -7,13 +7,11 @@ import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal- import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { AGENT_JOURNAL_THREAD_SCOPE, - type AgentJournalItemBody, type AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { partitionJournalLifecycleMutations } from '../agent-session-journal/journal-lifecycle-batch-partition' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' -import { cancelledJournalPromptBody } from '../agent-session-journal/journal-prompt-body-bounds' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { agentSessionFailureWords, @@ -26,15 +24,15 @@ import { } from './structured-agent-session-start-failure-row' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { - provenUnverifiableTurnRevisions, runningTurnLifecycleRevisions, - turnVerdictFromDeathEvidence, type StructuredAgentSessionTurnVerdict } from './structured-agent-session-stale-turn-verdict' +import { exitedRootTurnScope } from './structured-agent-session-exit-turn-scope' +import { openSettlementTerminalBody } from '../agent-session-journal/journal-open-settlement-plan' import { - exitedRootTurnScope, - runningRootTurnScope -} from './structured-agent-session-exit-turn-scope' + appendGoneGenerationSettlement, + planGoneGenerationSettlement +} from './structured-agent-session-open-settlement' /** Bounds the exit reason the lease keeps as log evidence; a provider diagnostic is held to the * same cap. */ @@ -169,7 +167,7 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { } for (const item of items) { const identity = parseAgentJournalItemKey(item.itemId) - const body = terminalDeadGenerationBody(item) + const body = openSettlementTerminalBody(item) if (identity && body) { mutations.push({ kind: 'item', @@ -197,11 +195,9 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { } /** - * Settles whatever a generation with no child in this process left running: found when a new child - * is acquired, or when a chat is reopened for reading. Derived from the journal and the lease's - * death evidence each time, so nothing is owed in between. Proven death ends the turn interrupted, - * and a proof written after an earlier settle revises what that settle left `unverifiable`. Must - * run before a new child's buffered events land, or a live turn would be judged. + * Settles whatever a generation with no child in this process left running, against the + * generation an acquisition just took (see `planGoneGenerationSettlement`, the open's own rule). + * Must run before a new child's buffered events land, or a live turn would be judged. */ export async function settleStaleStructuredAgentSessionState(input: { journal: AgentSessionJournal @@ -213,82 +209,24 @@ export async function settleStaleStructuredAgentSessionState(input: { failureTextContext?: AgentSessionFailureWordsContext }): Promise { const { journal } = input - const items = journal.snapshot().items - // Each turn is judged by the evidence only if it names that turn's owner. - const verdictFor = (item: AgentJournalRenderItem) => - turnVerdictFromDeathEvidence(input.deathEvidence, journal.itemFence(item.itemId)) - // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. - const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` - const settlementId = `stale-session:${input.sessionId}:${input.fence}:${generation}` - const mutations: JournalLifecycleMutationInput[] = [] - for (const item of items) { - const identity = parseAgentJournalItemKey(item.itemId) - const body = terminalDeadGenerationBody(item) - if (identity && body) { - mutations.push({ - kind: 'item', - identity, - body, - turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE - }) - } - } - const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal) - mutations.push( - ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), - ...proven - ) - const evidence = input.deathEvidence - if ( - evidence && - (proven.length > 0 || - items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted')) - ) { - mutations.unshift({ - kind: 'item', - // Named by the death it explains, so a retry after a partly written settle adds no second row. - identity: { - provider: 'orca', - clientMessageId: `stale-session:${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` - }, - // The death evidence is Orca's log text, never a sentence for a person: the row says only - // that the provider stopped. - body: { - kind: 'status', - ...agentSessionFailureWords(agentSessionFailureFact('providerExited'), { - ...input.failureTextContext, - surface: 'row' - }), - tone: 'error' - }, - turnScope: runningRootTurnScope(items) - }) - } - for (const chunk of partitionJournalLifecycleMutations(settlementId, mutations)) { - await journal.appendLifecycleBatch({ - settlementId: chunk.settlementId, - fence: input.fence, - recovered: true, - mutations: chunk.mutations - }) - } - return mutations.length -} - -function terminalDeadGenerationBody(item: AgentJournalRenderItem): AgentJournalItemBody | null { - if (item.body.kind === 'tool-call' && item.body.state === 'running') { - return { ...item.body, state: 'failed' } - } - if (item.body.kind === 'approval' || item.body.kind === 'question') { - return item.body.resolution.state === 'pending' ? cancelledJournalPromptBody(item.body) : null - } - return null + const settlement = planGoneGenerationSettlement({ + items: journal.snapshot().items, + itemFence: (itemId) => journal.itemFence(itemId), + sessionId: input.sessionId, + fence: input.fence, + // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. + generation: input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}`, + deathEvidence: input.deathEvidence, + failureTextContext: input.failureTextContext + }) + await appendGoneGenerationSettlement(journal, settlement, input.fence) + return settlement.mutations.length } function isUnfinishedItem(item: AgentJournalRenderItem): boolean { return ( readAgentJournalTurn(item.body)?.state === 'running' || - terminalDeadGenerationBody(item) !== null + openSettlementTerminalBody(item) !== null ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index c8f7e7f94b16..c1cbfbef32d0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -142,7 +142,8 @@ export class StructuredAgentSessionHost { isDisposed: () => this.lifetime.isDisposed(), // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. - onReadable: this.conversationDelivery.adoptOpened + onReadable: this.conversationDelivery.adoptOpened, + seedStatus: this.clientDelivery.seedStatus }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -235,6 +236,9 @@ export class StructuredAgentSessionHost { restoreReadableSessions = (sessionIds?: readonly string[]): Promise => this.restore.restoreReadableSessions(sessionIds) + // Startup, from each chat's stored state: see `structured-agent-session-startup-state`. + seedStoredStatuses = (ids: readonly string[]) => this.restore.seedStoredStatuses(ids) + settleOwedSessions = (ids: readonly string[]) => this.restore.settleOwedSessions(ids) /** Make one persisted session addressable again; see `structured-agent-session-reveal`. */ revealSession = (sessionId: string): Promise => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts new file mode 100644 index 000000000000..d1dea753aab0 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts @@ -0,0 +1,259 @@ +// Everything a non-acquisition open writes because an earlier host process is gone, as one plan. +// +// Built from the per-entity rules in journal-open-settlement-plan.ts, which the stored "owes work" +// flag reads too, so a chat's flag says it owes work exactly when this plan is non-empty. The open +// appends exactly this plan, and each entry revises its entity out of the state that selected it. + +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { + agentSessionFailureWords, + type AgentSessionFailureWordsContext +} from '../../../shared/agent-session-failure-words' +import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalRenderItem +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import { partitionJournalLifecycleMutations } from '../agent-session-journal/journal-lifecycle-batch-partition' +import { + isLeftoverQueuedSubmission, + isRunningJournalTurn, + openSettlementTerminalBody, + owesRecoveredDispatch +} from '../agent-session-journal/journal-open-settlement-plan' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + staleSubagentRosterRevisions, + type JournalSubagentLivenessRevision +} from '../agent-session-journal/journal-subagent-liveness' +import { runningRootTurnScope } from './structured-agent-session-exit-turn-scope' +import { + provenUnverifiableTurnRevisions, + runningTurnLifecycleRevisions, + turnVerdictFromDeathEvidence +} from './structured-agent-session-stale-turn-verdict' + +/** What the record says about the gone process, or null for the journal-only reading. */ +export type OpenSettlementRecordFacts = { + sessionId: string + fence: number + deathEvidence: AgentSessionDeathEvidence | null + /** Who the exit row names. */ + failureTextContext?: AgentSessionFailureWordsContext +} + +export type GoneGenerationSettlement = { + settlementId: string + mutations: JournalLifecycleMutationInput[] +} + +export type OpenSettlementPlan = { + /** Handed-over sends nothing answered: each becomes a recovered `unknown`. */ + recoveredDispatches: string[] + /** Sends accepted and never handed over: each is rejected as the host having restarted. */ + leftoverQueued: string[] + /** What the gone generation left running. Null on an acquisition's own open, which settles it + * against the generation it acquires. */ + goneGeneration: GoneGenerationSettlement | null + /** Rosters and background tasks still claiming a live child. */ + rosters: JournalSubagentLivenessRevision[] +} + +export type OpenSettlementJournal = Pick< + AgentSessionJournal, + 'snapshot' | 'submissions' | 'itemFence' | 'cursor' +> + +export type OpenSettlementOptions = { + acquisition?: boolean + /** False on a corrupt load, which still owes a rebuild that a roster write would retire. */ + settlesRosters: boolean +} + +export function planOpenSettlement( + journal: OpenSettlementJournal, + record: OpenSettlementRecordFacts | null, + options: OpenSettlementOptions +): OpenSettlementPlan { + const { items } = journal.snapshot() + const submissions = journal.submissions() + return { + recoveredDispatches: submissions + .filter(owesRecoveredDispatch) + .map((submission) => submission.clientMessageId), + leftoverQueued: submissions + .filter(isLeftoverQueuedSubmission) + .map((submission) => submission.clientMessageId), + goneGeneration: options.acquisition + ? null + : planGoneGenerationSettlement({ + items, + itemFence: (itemId) => journal.itemFence(itemId), + sessionId: record?.sessionId ?? '', + fence: record?.fence ?? 0, + // Per attempt: a retry re-partitions only what is left, and a reused chunk id would skip it. + generation: `seq-${journal.cursor().sequence}`, + deathEvidence: record?.deathEvidence ?? null, + failureTextContext: record?.failureTextContext + }), + rosters: options.settlesRosters ? staleSubagentRosterRevisions(items) : [] + } +} + +export function openSettlementPlanIsEmpty(plan: OpenSettlementPlan): boolean { + return ( + plan.recoveredDispatches.length === 0 && + plan.leftoverQueued.length === 0 && + (plan.goneGeneration?.mutations.length ?? 0) === 0 && + plan.rosters.length === 0 + ) +} + +/** + * Settles whatever a generation with no child in this process left running: found when a new child + * is acquired, or when a chat is reopened. Derived from the journal and the lease's death evidence + * each time, so nothing is owed in between. Proven death ends the turn interrupted, and a proof + * written after an earlier settle revises what that settle left `unverifiable`. + */ +export function planGoneGenerationSettlement(input: { + items: readonly AgentJournalRenderItem[] + itemFence: (itemId: string) => number | undefined + sessionId: string + fence: number + generation: string + deathEvidence: AgentSessionDeathEvidence | null + failureTextContext?: AgentSessionFailureWordsContext +}): GoneGenerationSettlement { + const { items } = input + // Each turn is judged by the evidence only if it names that turn's owner. + const verdictFor = (item: AgentJournalRenderItem) => + turnVerdictFromDeathEvidence(input.deathEvidence, input.itemFence(item.itemId)) + const mutations: JournalLifecycleMutationInput[] = [] + for (const item of items) { + const identity = parseAgentJournalItemKey(item.itemId) + const body = openSettlementTerminalBody(item) + if (identity && body) { + mutations.push({ + kind: 'item', + identity, + body, + turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE + }) + } + } + const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, { + itemFence: input.itemFence + }) + mutations.push( + ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), + ...proven + ) + const evidence = input.deathEvidence + if ( + evidence && + (proven.length > 0 || + items.some((item) => isInProgressItem(item) && verdictFor(item).state === 'interrupted')) + ) { + mutations.unshift({ + kind: 'item', + // Named by the death it explains, so a retry after a partly written settle adds no second row. + identity: { + provider: 'orca', + clientMessageId: `stale-session:${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` + }, + // The death evidence is Orca's log text, never a sentence for a person: the row says only + // that the provider stopped. + body: { + kind: 'status', + ...agentSessionFailureWords(agentSessionFailureFact('providerExited'), { + ...input.failureTextContext, + surface: 'row' + }), + tone: 'error' + }, + turnScope: runningRootTurnScope(items) + }) + } + return { + settlementId: `stale-session:${input.sessionId}:${input.fence}:${input.generation}`, + mutations + } +} + +/** Appends a gone generation's settlement, in bounded chunks each retry-safe by its id. */ +export async function appendGoneGenerationSettlement( + journal: Pick, + settlement: GoneGenerationSettlement, + fence: number +): Promise { + for (const chunk of partitionJournalLifecycleMutations( + settlement.settlementId, + settlement.mutations + )) { + await journal.appendLifecycleBatch({ + settlementId: chunk.settlementId, + fence, + recovered: true, + mutations: chunk.mutations + }) + } +} + +/** The open's writes: exactly the plan, in today's row shapes. Each part is best effort on its own: + * what fails stays owed, and the chat's next open or the next startup plans it again. */ +export async function appendOpenSettlement( + journal: AgentSessionJournal, + plan: OpenSettlementPlan, + fence: number, + onError: (error: unknown) => void +): Promise { + try { + for (const roster of plan.rosters) { + // The conversation's row, revised by whichever writer is newest. + await journal.appendItem(roster.identity, roster.body, { + fence: journal.highestFence(), + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + } + } catch (error) { + onError(error) + } + try { + // A handed-over row is only doubt, which provider history decides under a won lease. + if (plan.recoveredDispatches.length > 0) { + await journal.markPendingSubmissionsUnknown(fence) + } + // Nothing indexes a conversation with a row queued (a handle closes only with none), so one + // found here was accepted by a process that is gone. + if (plan.leftoverQueued.length > 0) { + const leftovers = new Set(plan.leftoverQueued) + await journal.rejectQueuedSubmissions( + fence, + agentSessionFailureWords(agentSessionFailureFact('hostRestarted'), { + surface: 'rejection' + }), + (submission) => leftovers.has(submission.clientMessageId) + ) + } + } catch (error) { + onError(error) + } + if (plan.goneGeneration && plan.goneGeneration.mutations.length > 0) { + try { + await appendGoneGenerationSettlement(journal, plan.goneGeneration, fence) + } catch (error) { + // Best effort: the next open or acquire re-derives it. + onError(error) + } + } +} + +/** Work that means the provider was MID-RESPONSE. A pending approval or question is the provider + * waiting on the user, so dying while one sits there interrupted nothing. */ +function isInProgressItem(item: AgentJournalRenderItem): boolean { + return ( + isRunningJournalTurn(item) || (item.body.kind === 'tool-call' && item.body.state === 'running') + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts index cf80396d8204..fd02951188ea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts @@ -1,6 +1,8 @@ import { existsSync } from 'node:fs' import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant' import { legacyJournalDatabaseFile } from '../agent-session-journal/journal-paths' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { JournalHostDatabase } from '../agent-session-journal/journal-host-database' import { readJournalSessionEpoch } from '../agent-session-journal/journal-row-table' import { openStructuredAgentSessionConversationJournal, @@ -22,21 +24,29 @@ export async function restoreStructuredAgentSessionRead( return null } const database = deps.journalDatabase - if (readJournalSessionEpoch(database.db, sessionId) === null) { - // Not in the host's database yet: its history may still sit in a per-chat file the open - // imports, or in the pre-SQLite format the open explains. - const legacyDirectory = database.legacyDirectoryFor({ - workspaceId: record.location.workspaceId, - sessionId - }) - if ( - !existsSync(legacyJournalDatabaseFile(legacyDirectory)) && - !findJournalFileFormatRemnant(legacyDirectory) - ) { - return null - } + if ( + readJournalSessionEpoch(database.db, sessionId) === null && + !hasHistoryOutsideJournalDatabase(database, record) + ) { + return null } return openStructuredAgentSessionConversationJournal(deps, record, { deferPerSessionImport: true }) } + +/** Not in the host's database yet, its history may still sit in a per-chat file the open imports, + * or in the pre-SQLite format the open explains. */ +export function hasHistoryOutsideJournalDatabase( + database: Pick, + record: Pick +): boolean { + const legacyDirectory = database.legacyDirectoryFor({ + workspaceId: record.location.workspaceId, + sessionId: record.sessionId + }) + return ( + existsSync(legacyJournalDatabaseFile(legacyDirectory)) || + findJournalFileFormatRemnant(legacyDirectory) !== null + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index ad03ad82e8cd..57bed2d2d809 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -14,7 +14,17 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { sessionTabListed } from './structured-agent-session-host-tabs' import { adapterSupportsRecord } from './structured-agent-session-provider-support' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' +import { + restoreStructuredAgentSessionsOnRestart, + type StructuredAgentSessionReadRestoreDeps +} from './structured-agent-session-restart-restore' +import { + createStructuredAgentSessionStartupState, + type StructuredAgentSessionStartupState, + type StructuredAgentSessionStartupStateDeps +} from './structured-agent-session-startup-state' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' import { createReaderReconcile, @@ -57,27 +67,29 @@ export async function revealStructuredAgentSession( } } -/** The host's startup readable-restore sweep: reconcile, resolve, then open each chat's journal. - * Its lease bookkeeping is a reader's, which never fails a read or startup; startup shares it. */ +/** The host's startup restore: reconcile, then seed and settle from the state stored beside each + * journal, then open in the background what that state cannot answer. Its lease bookkeeping is a + * reader's, which never fails a read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< - ConstructorParameters[0], - 'openDeps' | 'supportsRecord' | 'reconcile' | 'resolveRecovery' | 'isListed' + StructuredAgentSessionReadRestoreDeps, + 'openDeps' | 'reconcile' | 'resolveRecovery' | 'isListed' > & { reconcileLeases: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise + seedStatus: StructuredAgentSessionStartupStateDeps['seedStatus'] } ): { reconcileRestartLeases: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise -} { - const { reconcileLeases, resolveRecovery, ...rest } = wiring +} & StructuredAgentSessionStartupState { + const { reconcileLeases, resolveRecovery, seedStatus, ...rest } = wiring const failures = reportEachFailureOnce(deps.onLeaseReconcileFailure) const reconcile = createReaderReconcile(reconcileLeases, failures) - const restorer = new StructuredAgentSessionReadableRestorer({ + const supportsRecord = (record: AgentSessionRecord) => adapterSupportsRecord(deps.adapter, record) + const readRestore: StructuredAgentSessionReadRestoreDeps = { openDeps: deps, - supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), isListed: (sessionId) => sessionTabListed(deps.store, sessionId), reconcile, // The next attach or send resolves recovery again, strictly, before it acts. @@ -90,12 +102,23 @@ export function createStructuredAgentSessionHostRestore( } ), ...rest - }) + } + const restorer = new StructuredAgentSessionReadableRestorer({ ...readRestore, supportsRecord }) const gate = new StructuredAgentSessionRestartRestoreGate() return { reconcileRestartLeases: async () => { await reconcile('startup') }, - restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)) + restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), + ...createStructuredAgentSessionStartupState({ + openDeps: deps, + supportsRecord, + seedStatus, + restoreListed: (records) => + restoreStructuredAgentSessionsOnRestart({ ...readRestore, records }), + serialize: rest.serialize, + hasSession: rest.hasSession, + isDisposed: rest.isDisposed + }) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts new file mode 100644 index 000000000000..e47a1f171227 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -0,0 +1,181 @@ +// What host startup does with the state stored beside each chat's journal, before any client +// lists a tab, and without opening a chat that owes nothing. +// +// Seed: every settled, listed chat's status row is published from its stored state, so session +// lists have it at paint. Settle: every chat whose stored state says its open owes work, listed or +// not, is opened once, one at a time, and its open appends the settlement plan. A listed one goes +// through the restart restore's own per-chat worker and stays open; an unlisted one is settled and +// closed, never indexed or published. Everything else a listed chat needs (a stale row, a per-chat +// file not yet copied, a draft the drain would send) is left to the background restore after the +// listing, which is the same worker. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { owesOnOpen } from '../agent-session-journal/journal-open-settlement-plan' +import { + readJournalSessionStatesAtTip, + readOwedJournalSessionStates, + readSessionsWithDrainableDrafts +} from '../agent-session-journal/journal-session-state' +import { + openStructuredAgentSessionConversationJournal, + type StructuredAgentSessionConversationOpenDeps +} from './structured-agent-session-conversation-open' +import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' + +export type StructuredAgentSessionStartupStateDeps = { + openDeps: StructuredAgentSessionConversationOpenDeps & { + store: Pick + } + supportsRecord: (record: AgentSessionRecord) => boolean + seedStatus: ( + record: AgentSessionRecord, + stored: { projected: StructuredAgentSessionStatusProjection; lastActivityAt: number } + ) => void + /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish). */ + restoreListed: (records: AgentSessionRecord[]) => Promise + serialize: (sessionId: string, task: () => Promise) => Promise + hasSession: (sessionId: string) => boolean + isDisposed: () => boolean +} + +export type StructuredAgentSessionStartupState = { + /** Seeds settled listed chats; answers the listed ids the background restore still opens. */ + seedStoredStatuses: (listedIds: readonly string[]) => string[] + /** Settles every chat that owes work, once per host. Never rejects. */ + settleOwedSessions: (listedIds: readonly string[]) => Promise +} + +export function createStructuredAgentSessionStartupState( + deps: StructuredAgentSessionStartupStateDeps +): StructuredAgentSessionStartupState { + let settling: Promise | null = null + return { + seedStoredStatuses: (listedIds) => seedStoredStatuses(deps, listedIds), + settleOwedSessions: (listedIds) => { + settling ??= settleOwedSessions(deps, listedIds) + return settling + } + } +} + +function seedStoredStatuses( + deps: StructuredAgentSessionStartupStateDeps, + listedIds: readonly string[] +): string[] { + const database = deps.openDeps.journalDatabase + // A newer build's database: nothing is stored this build can trust, and every chat reads as it does. + if (database.readOnly) { + return [...listedIds] + } + let states: ReturnType + let drafts: Set + try { + states = readJournalSessionStatesAtTip(database.db, listedIds) + drafts = readSessionsWithDrainableDrafts(database.db) + } catch (error) { + // Fails open: every listed chat is restored in the background, as before stored state existed. + console.warn('[structured-agent-session] reading stored chat state failed', error) + return [...listedIds] + } + const byId = new Map(states.map((state) => [state.sessionId, state.current])) + const background: string[] = [] + for (const sessionId of listedIds) { + const record = deps.openDeps.store.getRecord(sessionId) + if (!record || !deps.supportsRecord(record)) { + continue + } + if (!byId.has(sessionId)) { + // Never sent opens nothing; a per-chat file not yet copied is read in the background. + if (hasHistoryOutsideJournalDatabase(database, record)) { + background.push(sessionId) + } + continue + } + const stored = byId.get(sessionId) + if (stored && owesOnOpen(stored, record.lease.deathEvidence)) { + // The settle step's: its open settles and publishes. + continue + } + const status = stored?.summary?.status + if (!stored?.summary || (status !== null && status !== 'idle')) { + // Stale, or describing a state nothing owed cannot reach: re-derived by its open. + background.push(sessionId) + continue + } + deps.seedStatus(record, { projected: stored.summary, lastActivityAt: stored.lastActivityAt }) + if (drafts.has(sessionId)) { + // Opened so the drain runs on open, as the restore always did. + background.push(sessionId) + } + } + return background +} + +async function settleOwedSessions( + deps: StructuredAgentSessionStartupStateDeps, + listedIds: readonly string[] +): Promise { + try { + const database = deps.openDeps.journalDatabase + if (database.readOnly) { + return + } + const listedOrder = new Map(listedIds.map((sessionId, index) => [sessionId, index])) + const listed: AgentSessionRecord[] = [] + const unlisted: AgentSessionRecord[] = [] + for (const owed of readOwedJournalSessionStates(database.db)) { + const record = deps.openDeps.store.getRecord(owed.sessionId) + if ( + !record || + !deps.supportsRecord(record) || + !owesOnOpen(owed, record.lease.deathEvidence) + ) { + continue + } + if (listedOrder.has(record.sessionId)) { + listed.push(record) + } else { + unlisted.push(record) + } + } + listed.sort( + (left, right) => + (listedOrder.get(left.sessionId) ?? 0) - (listedOrder.get(right.sessionId) ?? 0) + ) + await deps.restoreListed(listed) + for (const record of unlisted) { + // A journal open is synchronous SQLite: one chat per macrotask. + await yieldToEventLoop() + if (deps.isDisposed()) { + return + } + await deps + .serialize(record.sessionId, () => settleUnlisted(deps, record)) + .catch((error: unknown) => { + console.warn('[structured-agent-session] settling a chat at startup failed', { + sessionId: record.sessionId, + error + }) + }) + } + } catch (error) { + console.warn('[structured-agent-session] settling chats at startup failed', error) + } +} + +/** A chat with no tab: settled and closed, never indexed, so it gets no status row. */ +async function settleUnlisted( + deps: StructuredAgentSessionStartupStateDeps, + record: AgentSessionRecord +): Promise { + if (deps.isDisposed() || deps.hasSession(record.sessionId)) { + return + } + const opened = await openStructuredAgentSessionConversationJournal(deps.openDeps, record, { + deferPerSessionImport: true + }) + await opened.session.journal.close() +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index 22a61a006bb6..efc1def8958e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -7,26 +7,27 @@ // // The last projection is kept after the session's provider child is evicted: an idle session is // still idle without a process, and a renderer that reloads must not lose every settled row until -// each chat is reopened. Restart is the one boundary that forgets, and restoring readable sessions -// republishes them. +// each chat is reopened. At startup, a settled chat's row is seeded from the state stored beside its +// journal (journal-session-state.ts) without opening it; a chat that owes work is settled by its +// open, which publishes as any open does. -import { agentProviderSessionsEqual } from '../../../shared/agent-session-resume' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { normalizeOptionalField } from '../../../shared/agent-status-field-normalization' -import { isAgentStatusHeldOpenByChildWork } from '../../../shared/agent-lead-status-fold' -import { AGENT_MODEL_MAX_LENGTH } from '../../../shared/agent-status-types' -import { - agentSessionBackgroundTasksEqual, - type AgentSessionBackgroundTaskState, - type AgentSessionStatusEvent, - type AgentSessionStatusSummary +import type { + AgentSessionBackgroundTaskState, + AgentSessionStatusEvent, + AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import type { AgentChildWorkEvidence } from '../../../shared/agent-status-child-work-evidence' -import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' -import { structuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status' +import { + projectStructuredAgentSessionStatusState, + type StructuredAgentSessionStatusProjection +} from '../../../shared/structured-agent-session-projection' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' -import { structuredAgentSessionProviderSessionMetadata } from './structured-agent-session-history-result' +import { + structuredAgentSessionStatusSummary, + structuredAgentSessionSummariesEqual +} from './structured-agent-session-status-summary' import { StructuredAgentSessionStatusOwnership, type StructuredAgentSessionStatusSink @@ -66,47 +67,6 @@ export type StructuredAgentSessionStatusFeedDeps = { onAgentStarted?: (sessionId: string) => void } -function summariesEqual(a: AgentSessionStatusSummary, b: AgentSessionStatusSummary): boolean { - return ( - a.workspaceId === b.workspaceId && - a.agent === b.agent && - a.status === b.status && - a.hostExecutionOwned === b.hostExecutionOwned && - a.hostExecutionPhase === b.hostExecutionPhase && - a.hostExecutionChild?.generation === b.hostExecutionChild?.generation && - a.hostExecutionChild?.fence === b.hostExecutionChild?.fence && - a.rewindBlockedReason === b.rewindBlockedReason && - // A moved state clock changes ranking; row activity alone, including a subagent's, does not. - // An idle state the journal cannot date still republishes, since readers date it by `updatedAt`, - // and so does one live child work holds open: readers take each publish as its evidence. - a.statusStartedAt === b.statusStartedAt && - (a.status !== 'idle' || - a.updatedAt === b.updatedAt || - (a.statusStartedAt !== undefined && !isIdleHeldOpenByChildWork(b))) && - a.latestPrompt === b.latestPrompt && - a.model === b.model && - a.toolName === b.toolName && - a.toolInput === b.toolInput && - a.lastAssistantMessage === b.lastAssistantMessage && - a.turnOutcome === b.turnOutcome && - agentSessionBackgroundTasksEqual(a.backgroundTasks, b.backgroundTasks) && - agentProviderSessionsEqual(undefined, a.providerSession, b.providerSession) - ) -} - -function isIdleHeldOpenByChildWork(summary: AgentSessionStatusSummary): boolean { - return ( - summary.status === 'idle' && - isAgentStatusHeldOpenByChildWork( - structuredAgentSessionAgentStatus({ - status: summary.status, - backgroundTasks: summary.backgroundTasks, - turnOutcome: summary.turnOutcome - }) - ) - ) -} - /** Wire the host's own deps into a feed; keeps the host at one call site. * `deps` is a thunk because the host builds the feed in a field initializer, * before its constructor parameters are assigned. */ @@ -236,22 +196,34 @@ export class StructuredAgentSessionStatusFeed { if (!session) { return } - const summary = this.summaryFor(sessionId, session, journal ?? session.journal) + this.publishSummary( + this.summaryFor(sessionId, session, journal ?? session.journal), + session.params.location, + { replay: options?.replay === true } + ) + } + + private publishSummary( + summary: AgentSessionStatusSummary, + location: AgentSessionRecord['location'], + options: { replay: boolean } + ): void { + const { sessionId } = summary const previous = this.published.get(sessionId) - if (previous && summariesEqual(previous, summary)) { - if (!this.ownership.matchesLocation(sessionId, session.params.location)) { - this.sink(summary, session.params.location) + if (previous && structuredAgentSessionSummariesEqual(previous, summary)) { + if (!this.ownership.matchesLocation(sessionId, location)) { + this.sink(summary, location) } return } this.published.set(sessionId, summary) - this.sink(summary, session.params.location) + this.sink(summary, location) this.broadcast({ type: 'status', session: summary }) if (summary.hostExecutionPhase === 'ready' && previous?.hostExecutionPhase !== 'ready') { this.deps.onAgentStarted?.(sessionId) } try { - this.deps.onStatusChanged?.(summary, { replay: options?.replay === true }) + this.deps.onStatusChanged?.(summary, options) } catch (error) { // An observer must never cost the subscribers their status event. console.warn('[structured-session-status] status observer failed', error) @@ -264,37 +236,45 @@ export class StructuredAgentSessionStatusFeed { journal: AgentSessionJournal ): AgentSessionStatusSummary { const record = this.deps.getRecord(sessionId) - const { summary: projected } = this.projectionFor(journal, record) - const providerSession = structuredAgentSessionProviderSessionMetadata(record) - // The journal has no model: the record's acknowledged options are where a mid-session - // switch lands, so the row follows whichever is in force. - const model = normalizeOptionalField(record?.options?.model, AGENT_MODEL_MAX_LENGTH) - // Usage is dropped here on purpose: a `task_progress` tick would otherwise fail the - // equality check and re-broadcast a full summary to every remote subscriber for a - // number no session list renders. Tokens stay live on the background-task channel. - const backgroundTasks = this.deps - .readBackgroundTasks?.(sessionId) - ?.tasks?.map(({ totalTokens: _totalTokens, ...task }) => task) - return { + return structuredAgentSessionStatusSummary({ sessionId, - workspaceId: session.params.location.workspaceId, - agent: session.params.provider, - ...(session.child - ? { - hostExecutionOwned: true as const, - hostExecutionPhase: session.child.phase, - hostExecutionChild: { generation: session.child.generation, fence: session.child.fence } - } - : {}), - ...projected, - ...(record?.rewind?.phase === 'prepared' || record?.rewind?.phase === 'provider-succeeded' - ? { rewindBlockedReason: 'outcome-unknown' as const } - : {}), - ...(model ? { model } : {}), - ...(backgroundTasks && backgroundTasks.length > 0 ? { backgroundTasks } : {}), - ...(providerSession ? { providerSession } : {}), - updatedAt: journal.lastActivityAt() || this.deps.now() + params: session.params, + record, + child: session.child, + projected: this.projectionFor(journal, record).summary, + backgroundTasks: this.deps.readBackgroundTasks?.(sessionId), + lastActivityAt: journal.lastActivityAt(), + now: this.deps.now + }) + } + + /** + * A chat's row from the state stored beside its journal, for one this host has not opened: the + * same builder an open's publish uses, so the open later finds it equal and sends nothing. A + * replay, as a restore's publish is. + */ + seed( + record: AgentSessionRecord, + stored: { projected: StructuredAgentSessionStatusProjection; lastActivityAt: number } + ): void { + const { sessionId } = record + if (this.deps.sessions.has(sessionId)) { + return } + const params = { location: record.location, provider: record.provider } + this.publishSummary( + structuredAgentSessionStatusSummary({ + sessionId, + params, + record, + projected: stored.projected, + backgroundTasks: this.deps.readBackgroundTasks?.(sessionId), + lastActivityAt: stored.lastActivityAt, + now: this.deps.now + }), + params.location, + { replay: true } + ) } /** Child-work evidence for a session this feed publishes; a failing sink costs nothing else. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts new file mode 100644 index 000000000000..ddc6561166c5 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-summary.ts @@ -0,0 +1,106 @@ +// One structured session's status row, built the same way whether its projected fields come from +// an open journal or from the state stored beside it (journal-session-state.ts), so a row seeded +// at startup and the row the chat's open later publishes are equal field for field. + +import { agentProviderSessionsEqual } from '../../../shared/agent-session-resume' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { normalizeOptionalField } from '../../../shared/agent-status-field-normalization' +import { isAgentStatusHeldOpenByChildWork } from '../../../shared/agent-lead-status-fold' +import { AGENT_MODEL_MAX_LENGTH } from '../../../shared/agent-status-types' +import { + agentSessionBackgroundTasksEqual, + type AgentSessionBackgroundTaskState, + type AgentSessionStatusSummary +} from '../../../shared/agent-session-wire' +import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' +import { structuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status' +import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' +import { structuredAgentSessionProviderSessionMetadata } from './structured-agent-session-history-result' + +export function structuredAgentSessionStatusSummary(input: { + sessionId: string + params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } + record: AgentSessionRecord | null + child?: Pick | null + projected: StructuredAgentSessionStatusProjection + backgroundTasks?: AgentSessionBackgroundTaskState | null + /** The journal's newest activity; 0 when it can date none. */ + lastActivityAt: number + now: () => number +}): AgentSessionStatusSummary { + const { record, child } = input + const providerSession = structuredAgentSessionProviderSessionMetadata(record) + // The journal has no model: the record's acknowledged options are where a mid-session + // switch lands, so the row follows whichever is in force. + const model = normalizeOptionalField(record?.options?.model, AGENT_MODEL_MAX_LENGTH) + // Usage is dropped here on purpose: a `task_progress` tick would otherwise fail the + // equality check and re-broadcast a full summary to every remote subscriber for a + // number no session list renders. Tokens stay live on the background-task channel. + const backgroundTasks = input.backgroundTasks?.tasks?.map( + ({ totalTokens: _totalTokens, ...task }) => task + ) + return { + sessionId: input.sessionId, + workspaceId: input.params.location.workspaceId, + agent: input.params.provider, + ...(child + ? { + hostExecutionOwned: true as const, + hostExecutionPhase: child.phase, + hostExecutionChild: { generation: child.generation, fence: child.fence } + } + : {}), + ...input.projected, + ...(record?.rewind?.phase === 'prepared' || record?.rewind?.phase === 'provider-succeeded' + ? { rewindBlockedReason: 'outcome-unknown' as const } + : {}), + ...(model ? { model } : {}), + ...(backgroundTasks && backgroundTasks.length > 0 ? { backgroundTasks } : {}), + ...(providerSession ? { providerSession } : {}), + updatedAt: input.lastActivityAt || input.now() + } +} + +export function structuredAgentSessionSummariesEqual( + a: AgentSessionStatusSummary, + b: AgentSessionStatusSummary +): boolean { + return ( + a.workspaceId === b.workspaceId && + a.agent === b.agent && + a.status === b.status && + a.hostExecutionOwned === b.hostExecutionOwned && + a.hostExecutionPhase === b.hostExecutionPhase && + a.hostExecutionChild?.generation === b.hostExecutionChild?.generation && + a.hostExecutionChild?.fence === b.hostExecutionChild?.fence && + a.rewindBlockedReason === b.rewindBlockedReason && + // A moved state clock changes ranking; row activity alone, including a subagent's, does not. + // An idle state the journal cannot date still republishes, since readers date it by `updatedAt`, + // and so does one live child work holds open: readers take each publish as its evidence. + a.statusStartedAt === b.statusStartedAt && + (a.status !== 'idle' || + a.updatedAt === b.updatedAt || + (a.statusStartedAt !== undefined && !isIdleHeldOpenByChildWork(b))) && + a.latestPrompt === b.latestPrompt && + a.model === b.model && + a.toolName === b.toolName && + a.toolInput === b.toolInput && + a.lastAssistantMessage === b.lastAssistantMessage && + a.turnOutcome === b.turnOutcome && + agentSessionBackgroundTasksEqual(a.backgroundTasks, b.backgroundTasks) && + agentProviderSessionsEqual(undefined, a.providerSession, b.providerSession) + ) +} + +function isIdleHeldOpenByChildWork(summary: AgentSessionStatusSummary): boolean { + return ( + summary.status === 'idle' && + isAgentStatusHeldOpenByChildWork( + structuredAgentSessionAgentStatus({ + status: summary.status, + backgroundTasks: summary.backgroundTasks, + turnOutcome: summary.turnOutcome + }) + ) + ) +} diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index 02533588e9dd..1d383b985c15 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -289,6 +289,12 @@ async function startOrcadRuntime( // Recovery binds terminal and dispatch identities; only now can startup observations be fenced. observedStatusCapture.attach(runtime) + // Why here and not on a client's first listing: a headless host seeds its chats' statuses and + // settles crashed ones itself, before any client connects. + void runtime.prepareStructuredAgentSessionStartupRestoration().catch((error: unknown) => { + console.warn('[structured-agent-session] startup restoration failed', error) + }) + const bindHost = resolveOrcadBindHost(options.bind) rpc = new OrcaRuntimeRpcServer({ runtime, diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index b242208ad127..9266e19f4082 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -10,6 +10,7 @@ import { resolveStructuredAgentSessionAdoptionForCreate } from './structured-agent-session-create-adoption' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' +import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { getLocalProjectWorktreeGitOptions } from '../project-runtime-git-options' import type { AgentSessionAttachParams } from '../native-chat/agent-session-wire/structured-agent-session-attach' import { resolveTuiAgentLaunchEnv } from '../../shared/tui-agent-launch-defaults' @@ -28,6 +29,8 @@ import { applyStructuredCodexWorkspaceTrust } from '../agent-workspace-trust-spa export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaRuntimeWithGetWorktreePs { // The history restore a tab restore owes, until a caller that answered with its list starts it. protected owedStructuredAgentSessionHistoryRestore: (() => void) | null = null + // Listed chats startup could not answer from stored state; null until it has run. + protected structuredAgentSessionBackgroundRestoreIds: string[] | null = null async getStructuredAgentSessionCreateSupport( worktreeSelector: string, @@ -274,7 +277,38 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR this.ensureStructuredAgentSessionHost() ) await this.refreshMobileSessionPtyRecords() - await getStructuredAgentSessionHost()?.reconcileRestartLeases() + const host = getStructuredAgentSessionHost() + // Death evidence the lease check writes must exist before the settle takes its verdicts. A + // failed check leaves them `unverifiable`, which is safe, so the steps below still run. + let leaseFailure: { error: unknown } | null = null + try { + await host?.reconcileRestartLeases() + } catch (error) { + leaseFailure = { error } + } + if (typeof host?.seedStoredStatuses === 'function') { + const listedIds = this.listedStructuredAgentSessionIds(host) + this.structuredAgentSessionBackgroundRestoreIds = host.seedStoredStatuses(listedIds) + // Un-awaited: nothing waits on a crashed chat's settle, and it never rejects. + void host.settleOwedSessions(listedIds) + } + if (leaseFailure) { + throw leaseFailure.error + } + } + + /** The chats with a tab: the host's persisted tab index, or before it existed, the saved + * workspace session's. */ + protected listedStructuredAgentSessionIds(host): string[] { + const persistedVisibleIndex = + typeof host?.getPersistedVisibleSessionTabIndex === 'function' + ? host.getPersistedVisibleSessionTabIndex() + : { present: false, sessionIds: [] } + return persistedVisibleIndex.present + ? persistedVisibleIndex.sessionIds + : collectSavedStructuredAgentSessionIds( + this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + ) } protected hasPersistedStructuredAgentSessionStore(): boolean { diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 672cb9313574..eeb895bd7363 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -4,9 +4,7 @@ import { OrcaRuntimeWithGetStructuredAgentSessionCreateSupport } from './orca-ru import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' -import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { seedStructuredAgentSessionTabIndex } from './structured-agent-session-tab-index-seed' -import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { RuntimeMobileSessionAgentTab, RuntimeMobileSessionTabsSnapshot, @@ -53,15 +51,7 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu protected async restoreStructuredAgentSessionTabsOnce(): Promise { await this.prepareStructuredAgentSessionStartupRestoration() const host = getStructuredAgentSessionHost() - const persistedVisibleIndex = - typeof host?.getPersistedVisibleSessionTabIndex === 'function' - ? host.getPersistedVisibleSessionTabIndex() - : { present: false, sessionIds: [] } - const listedIds = persistedVisibleIndex.present - ? persistedVisibleIndex.sessionIds - : collectSavedStructuredAgentSessionIds( - this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null - ) + const listedIds = this.listedStructuredAgentSessionIds(host) for (const worktreeId of this.getKnownWorkspaceSessionWorktreeIds()) { this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { allowAttachedWindow: true, @@ -94,11 +84,13 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const session of restored) { this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } - // Each chat's status row arrives as its history opens; one chat's failure leaves the others and - // the list alone. Whoever answers with this list starts it, once that answer is out. + // Startup already seeded every settled chat's row and is settling the ones that owe work; this + // opens only what stored state could not answer (a stale row, a per-chat file not yet copied, a + // draft to drain). Whoever answers with this list starts it, once that answer is out. + const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds this.owedStructuredAgentSessionHistoryRestore = host ? () => - void host.restoreReadableSessions(listedIds).catch((error: unknown) => { + void host.restoreReadableSessions(background).catch((error: unknown) => { console.warn( '[structured-agent-session] restoring chat history after listing failed', error diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index 38e127dcf845..1e2d988c980f 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -312,7 +312,8 @@ describe('restoring the chat tabs open at quit', () => { expect(published()).toHaveLength(count) expect(prepared).toBe(1) - expect(lock.refused - prepared).toBe(1) + // Every chat's stored state answers it, so the background restore opens none and takes no lock. + expect(lock.refused - prepared).toBe(0) } ) @@ -362,8 +363,8 @@ describe('restoring the chat tabs open at quit', () => { expect(published()).toHaveLength(2) expect(prepared).toBe(1) - // The restore's lease check and the seed. - expect(lock.refused - prepared).toBe(2) + // The seed alone: stored state answers both chats, so the background restore takes no lock. + expect(lock.refused - prepared).toBe(1) }) it('still lists the chats when that write fails, and leaves the index absent', async () => { diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 6400a61903ae..a38a101e7b71 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -4,7 +4,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../shared/protocol-version' import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' -import { closeTestJournalHostDatabases } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../native-chat/agent-session-journal/journal-host-database-test-support' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { createRestTestRig, @@ -107,6 +110,11 @@ function restartedRuntime(workspaceSession: unknown = null) { return { runtime, listAll, subscribeAll } } +/** The files an older build left: no chat has stored state, so each is re-derived by its open. */ +function forgetStoredChatState(): void { + openTestJournalHostDatabase(rig.root).db.exec('DELETE FROM journal_session_state') +} + function chatIds(snapshots: RuntimeMobileSessionTabsResult[]): string[] { return snapshots.flatMap((snapshot) => snapshot.tabs.flatMap((tab) => (tab.type === 'agent-session' ? [tab.sessionId] : [])) @@ -129,6 +137,7 @@ describe('listing chat tabs at startup', () => { } await rig.crash() await rig.boot() + forgetStoredChatState() const opens = Promise.withResolvers() releaseHeld = opens.resolve let opened = 0 @@ -156,6 +165,7 @@ describe('listing chat tabs at startup', () => { } await rig.crash() await rig.boot() + forgetStoredChatState() const restore = vi.spyOn(rig.host, 'restoreReadableSessions') const { runtime, listAll, subscribeAll } = restartedRuntime() const seen = () => ({ @@ -295,6 +305,7 @@ describe('listing chat tabs at startup', () => { } await rig.crash() await rig.boot() + forgetStoredChatState() rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { if (failing.includes(sessionId)) { throw new Error('EACCES: permission denied') diff --git a/src/main/startup/main-process-runtime-launch.ts b/src/main/startup/main-process-runtime-launch.ts index b3df23e5861c..4b77d878cc5d 100644 --- a/src/main/startup/main-process-runtime-launch.ts +++ b/src/main/startup/main-process-runtime-launch.ts @@ -337,6 +337,13 @@ export async function initializeMainProcessRuntimeLaunch( await shellPathReady bindTerminalRuntimeStartupServices(Promise.resolve(startTerminalRuntimeStartupServices())) } + // The host seeds its chats' statuses and settles crashed ones itself, so a window, a remote + // client or neither sees the same state. Behind the barriers the renderer's own call awaits. + void Promise.all([state.firstWindowStartupServicesReady, state.managedWslCliStartupBarrierReady]) + .then(() => runtime.prepareStructuredAgentSessionStartupRestoration()) + .catch((error: unknown) => { + console.warn('[structured-agent-session] startup restoration failed', error) + }) app.on('activate', options.handleMacAppActivation) if (serveOptions) { await launchServeMode(runtime, runtimeRpc, serveOptions) From f2276858d057c854389b9e7357729564b491d107 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:49:38 -0700 Subject: [PATCH 019/107] test(native-chat): stored chat state, open settlement plan and startup step Covers the in-transaction fold and its recovery (T1a-c), a failing state write (T2), stored state equal to a fresh replay after every append (T3), flag and plan agreement (T4, T4b, T15b), seeded rows equal to an open's publish (T5), startup opening only what it must and settling crashed chats with or without a tab (T6, T7, T14), headless startup and first snapshots (T8, T17), older builds (T9), epoch/import/repair writes (T10), the versioned derivation (T11), tab close (T12), status observers (T13), fence independence (T15a), a newer database (T16), and failures in the step (T18). --- .../journal-append-fold-recovery.test.ts | 160 +++++++++ .../journal-per-session-import.test.ts | 26 ++ .../journal-session-state-test-corpus.ts | 197 +++++++++++ .../journal-session-state.test.ts | 296 ++++++++++++++++ ...ured-agent-session-open-settlement.test.ts | 140 ++++++++ ...ctured-agent-session-startup-state.test.ts | 315 ++++++++++++++++++ ...-agent-session-startup-restoration.test.ts | 121 +++++++ 7 files changed, 1255 insertions(+) create mode 100644 src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-session-state.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts create mode 100644 src/main/runtime/structured-agent-session-startup-restoration.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts b/src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts new file mode 100644 index 000000000000..e64345417028 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts @@ -0,0 +1,160 @@ +// An append folds its row inside the transaction, so the chat's stored state can describe it. A +// transaction that fails after that fold leaves the fold ahead of the disk: the chat re-folds from +// what committed, or closes when a stranded connection could still show it the uncommitted row. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { + createTrackedJournalOpener, + loadTestJournal, + openTestJournalHostDatabase +} from './journal-host-database-test-support' +import { applyJournalRow, renderJournalState } from './journal-reducer' +import type * as JournalReducer from './journal-reducer' +import { readJournalSessionState } from './journal-session-state' +import type { AgentSessionJournal } from './journal-store' + +vi.mock('./journal-reducer', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, applyJournalRow: vi.fn(actual.applyJournalRow) } +}) + +const SESSION = 'session-fold' +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: SESSION, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-fold' } +} + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +function open(): Promise { + return journals.open({ + identity: IDENTITY, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => 'epoch-fold' + }) +} + +function note(journal: AgentSessionJournal, id: string) { + return journal.appendItem( + { provider: 'orca', clientMessageId: id }, + { kind: 'status', text: id }, + { fence: 1, turnScope: { kind: 'thread' } } + ) +} + +/** The chat's fold, as a reader of it sees it. */ +function foldOf(journal: AgentSessionJournal) { + return { + cursor: journal.cursor(), + snapshot: journal.snapshot(), + lastActivityAt: journal.lastActivityAt(), + highestFence: journal.highestFence() + } +} + +/** The same, from a fresh replay of what is on disk. */ +function durableFold() { + const loaded = loadTestJournal(root, SESSION)! + return { + cursor: { epoch: loaded.state.epoch, sequence: loaded.state.lastSequence }, + snapshot: renderJournalState(loaded.state), + lastActivityAt: loaded.state.lastActivityAt, + highestFence: loaded.state.highestFence + } +} + +/** Fails the next statement(s) the connection runs whose SQL is in `failing`, once each. */ +function failOnce(failing: string[]): void { + const database = openTestJournalHostDatabase(root) + const connection = database.db + const exec = connection.exec.bind(connection) + const pending = new Set(failing) + vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { + if (pending.delete(sql)) { + throw new Error(`${sql} failed: disk I/O error`) + } + exec(sql) + }) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-fold-recovery-')) + clock = 1_000 +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe('an append whose transaction fails after its row folded', () => { + it('re-folds from disk when the fold itself throws part-way (T1a)', async () => { + const journal = await open() + await note(journal, 'first') + const stateBefore = readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION) + // One mutation lands, then the fold throws: the in-place apply cannot be undone. + vi.mocked(applyJournalRow).mockImplementationOnce((state, row) => { + state.lastSequence = row.seq + throw new Error('fold failed part-way') + }) + + await expect(note(journal, 'second')).rejects.toThrow('fold failed part-way') + + expect(durableFold().cursor.sequence).toBe(2) + expect(foldOf(journal)).toEqual(durableFold()) + expect(readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION)).toEqual( + stateBefore + ) + // The next append takes the sequence the failed one could not. + await expect(note(journal, 'third')).resolves.toMatchObject({ cursor: { sequence: 3 } }) + }) + + it('re-folds from disk when COMMIT fails and the rollback goes through (T1b)', async () => { + const journal = await open() + await note(journal, 'first') + const stateBefore = readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + failOnce(['COMMIT']) + + await expect(note(journal, 'second')).rejects.toThrow('COMMIT failed') + + expect(durableFold().cursor.sequence).toBe(2) + expect(foldOf(journal)).toEqual(durableFold()) + expect(readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION)).toEqual( + stateBefore + ) + await expect(note(journal, 'third')).resolves.toMatchObject({ cursor: { sequence: 3 } }) + }) + + it('closes the chat rather than re-reading a stranded connection (T1c)', async () => { + const journal = await open() + await note(journal, 'first') + const stranded = vi.fn() + journal.observeStranded(stranded) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + failOnce(['COMMIT', 'ROLLBACK']) + + await expect(note(journal, 'second')).rejects.toThrow('COMMIT failed') + + expect(stranded).toHaveBeenCalledOnce() + // Closed: no later write can land past a row that never committed. + await expect(note(journal, 'third')).rejects.toMatchObject({ code: 'journal_closed' }) + // The host database's own retry rolls the stranded transaction back on its next use. + expect(openTestJournalHostDatabase(root).db.isTransaction).toBe(false) + expect(durableFold().cursor.sequence).toBe(2) + const reopened = await open() + expect(foldOf(reopened)).toEqual(durableFold()) + await expect(note(reopened, 'fourth')).resolves.toMatchObject({ cursor: { sequence: 3 } }) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts index ea28ba4ef838..9f9d1b052c1d 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts @@ -27,6 +27,7 @@ import { } from './journal-host-database-test-support' import { journalDirectoryFor, legacyJournalDatabaseFile } from './journal-paths' import { importPerSessionJournal } from './journal-per-session-import' +import { readJournalSessionState } from './journal-session-state' import { readJournalSessionEpoch, type JournalStoredRow } from './journal-row-table' vi.mock('node:fs', async (importOriginal) => { @@ -208,6 +209,31 @@ describe('importing a per-chat journal', () => { expect(await leftovers()).toEqual([]) }) + // T10: the stored state described rows the copy replaced, so the copy's publish drops it. + it('drops the chat stored state in the copy publish, and nothing else writes one', async () => { + const { epoch, rows } = await historyRows() + await writeLegacyJournal(epoch, rows) + const { db } = openTestJournalHostDatabase(root) + db.prepare( + `INSERT INTO journal_session_state (session_id, state_version, epoch, seq, owes_work, + unverifiable_owner_fences, summary_json, last_activity_at, written_at) + VALUES (?, 1, 'stale-epoch', 9, 1, NULL, NULL, 1, 1)` + ).run(IDENTITY.sessionId) + + await importPerSessionJournal({ + database: openTestJournalHostDatabase(root), + identity: IDENTITY, + legacyDirectory: legacyDir() + }) + + expect(readJournalSessionState(db, IDENTITY.sessionId)).toBeNull() + expect( + db + .prepare('SELECT COUNT(*) AS n FROM journal_session_state WHERE session_id = ?') + .get(IDENTITY.sessionId) + ).toEqual({ n: 0 }) + }) + // T-B3: the upgrade restart is the restart that produced the offers. A new epoch or renumbered // rows would silently withdraw every "resume after update" offer. it('keeps a restart offer taken before the upgrade', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts new file mode 100644 index 000000000000..c0ff4d627095 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts @@ -0,0 +1,197 @@ +// Chats in every state an open's settlement plan can find, written through a real journal store, +// for the tests of the stored state and the plan that must agree about them. + +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import type { AgentSessionJournal } from './journal-store' + +/** The fence every case writes its content under; an `unverifiable` turn's writer. */ +export const CORPUS_FENCE = 3 + +const THREAD = 'thread-corpus' + +function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: THREAD, turnId, ordinal } +} + +async function item( + journal: AgentSessionJournal, + identity: AgentJournalItemIdentity, + body: AgentJournalItemBody +): Promise { + await journal.appendItem(identity, body, { + fence: CORPUS_FENCE, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) +} + +async function send( + journal: AgentSessionJournal, + clientMessageId: string, + text: string, + handoverRecorded?: true +): Promise { + await journal.appendSubmission({ + clientMessageId, + payloadFingerprint: `fp-${clientMessageId}`, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + fence: CORPUS_FENCE, + ...(handoverRecorded ? { handoverRecorded } : {}) + }) +} + +async function settledTurn(journal: AgentSessionJournal, turnId: string): Promise { + await send(journal, `send-${turnId}`, `asked ${turnId}`) + await journal.resolveDispatch({ + clientMessageId: `send-${turnId}`, + state: 'accepted', + providerIdentity: codexItem(turnId, 1), + fence: CORPUS_FENCE + }) + await item(journal, codexItem(turnId, 0), { + kind: 'turn', + turnId, + state: 'completed', + outcome: 'success', + startedAt: 10, + completedAt: 20 + }) + await item(journal, codexItem(turnId, 2), { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: `answered ${turnId}` }] + }) +} + +function roster(state: 'working' | 'done'): AgentJournalItemBody { + return { + kind: 'message', + role: 'system', + blocks: [ + { + type: 'subagent-group', + groupId: 'group-1', + agents: [{ id: 'child-1', label: 'reads', state, startedAt: 10 }] + } + ] + } +} + +/** Each case, written onto a freshly opened (empty) journal. */ +export const JOURNAL_SESSION_STATE_CORPUS = { + empty: async () => undefined, + settled: (journal: AgentSessionJournal) => settledTurn(journal, 'turn-1'), + 'older turn running beside a newer one': async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-0', 0), { + kind: 'turn', + turnId: 'turn-0', + state: 'running', + startedAt: 5 + }) + await settledTurn(journal, 'turn-1') + }, + 'working subagent roster': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('working')) + }, + 'live background task': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item( + journal, + { provider: 'orca', clientMessageId: 'claude-background-task:task-1' }, + { + kind: 'message', + role: 'system', + blocks: [ + { + type: 'background-task', + taskId: 'task-1', + kind: 'command', + label: 'sleep 20', + state: 'working', + startedAt: 10 + } + ] + } + ) + }, + 'pending prompt': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-1', 3), { + kind: 'approval', + title: 'Approve?', + detail: null, + options: [], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }) + }, + 'running tool': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-1', 4), { + kind: 'tool-call', + name: 'shell', + input: { command: 'ls' }, + state: 'running' + }) + }, + 'pending send': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-pending', 'never answered') + }, + 'unknown send': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-unknown', 'ambiguous') + await journal.resolveDispatch({ + clientMessageId: 'send-unknown', + state: 'unknown', + reason: 'adapter_timeout', + fence: CORPUS_FENCE + }) + }, + 'queued leftover': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-queued', 'waiting its turn', true) + }, + 'unverifiable turn': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-2', 0), { + kind: 'turn', + turnId: 'turn-2', + state: 'unverifiable', + startedAt: 30 + }) + }, + 'settled roster': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('done')) + } +} satisfies Record Promise> + +export type JournalSessionStateCase = keyof typeof JOURNAL_SESSION_STATE_CORPUS + +export const JOURNAL_SESSION_STATE_CASES = Object.keys(JOURNAL_SESSION_STATE_CORPUS).filter( + (name): name is JournalSessionStateCase => name in JOURNAL_SESSION_STATE_CORPUS +) + +/** Death evidence as a record can carry it: none, an older build's (no owner), naming the + * corpus's writer, and naming another owner. */ +export const CORPUS_DEATH_EVIDENCE: Record = { + none: null, + 'older build, no owner': { kind: 'exit-observed', detail: 'exit', observedAt: 50 }, + 'names the writer': { + kind: 'pid-absent', + detail: 'gone', + observedAt: 60, + ownerFence: CORPUS_FENCE + }, + 'names another owner': { + kind: 'exit-observed', + detail: 'exit', + observedAt: 70, + ownerFence: CORPUS_FENCE + 1 + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts new file mode 100644 index 000000000000..24b5142808b6 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -0,0 +1,296 @@ +// Each chat's stored state: written in the journal row's own transaction, equal to what a fresh +// replay derives, never able to fail an append, and trusted only at the chat's live tip. + +import { createHash } from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' +import { journalPragmaNumber } from './journal-database' +import { + createTrackedJournalOpener, + insertTestJournalRow, + loadTestJournal, + openTestJournalHostDatabase +} from './journal-host-database-test-support' +import { renderJournalState } from './journal-reducer' +import { deleteJournalRepairedSuffix } from './journal-repair-marker' +import { + deriveJournalSessionState, + JOURNAL_SESSION_STATE_VERSION, + readJournalSessionState, + readJournalSessionStatesAtTip, + type DerivedJournalSessionState +} from './journal-session-state' +import { + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS, + type JournalSessionStateCase +} from './journal-session-state-test-corpus' +import type { AgentSessionJournal } from './journal-store' + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +function identity(sessionId: string): AgentSessionJournalIdentity { + return { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + } +} + +function open(sessionId: string): Promise { + return journals.open({ + identity: identity(sessionId), + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}-${clock}` + }) +} + +function db() { + return openTestJournalHostDatabase(root).db +} + +/** What a fresh open of the chat would derive, read back from disk. */ +function freshDerivation(sessionId: string): DerivedJournalSessionState { + const loaded = loadTestJournal(root, sessionId) + if (!loaded) { + throw new Error(`no journal for ${sessionId}`) + } + return deriveJournalSessionState(loaded.state, { settlesRosters: !loaded.corrupt }) +} + +function stored(sessionId: string) { + const row = readJournalSessionState(db(), sessionId) + if (!row) { + return null + } + const { stateVersion: _stateVersion, writtenAt: _writtenAt, ...derived } = row + return derived +} + +async function write(name: JournalSessionStateCase): Promise { + const journal = await open(name) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + return journal +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-session-state-')) + clock = 1_000 +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe('the stored state follows every append (T3)', () => { + it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { + const journal = await open(name) + const mismatches: string[] = [] + // Told after each commit: the row and its state landed in one transaction. + journal.observeCommits(() => { + const expected = freshDerivation(name) + if (!isDeepStrictEqual(stored(name), expected)) { + mismatches.push(`seq ${expected.seq}`) + } + }) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + await Promise.resolve() + expect(mismatches).toEqual([]) + expect(stored(name)).toEqual(freshDerivation(name)) + }) +}) + +describe('a failing state write never fails the append (T2)', () => { + it('commits the row, leaves the state behind the tip, and the next open re-derives it', async () => { + const journal = await write('settled') + const before = stored('settled') + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + db().exec(`CREATE TEMP TRIGGER fail_state_write BEFORE UPDATE ON main.journal_session_state + BEGIN SELECT RAISE(ABORT, 'state write refused'); END`) + + await expect( + journal.appendItem( + { provider: 'orca', clientMessageId: 'note-1' }, + { kind: 'status', text: 'a note' }, + { fence: 3, turnScope: { kind: 'thread' } } + ) + ).resolves.toMatchObject({ cursor: { sequence: before!.seq + 1 } }) + + expect(loadTestJournal(root, 'settled')?.state.lastSequence).toBe(before!.seq + 1) + expect(stored('settled')).toEqual(before) + expect(readJournalSessionStatesAtTip(db(), ['settled'])).toEqual([ + { sessionId: 'settled', current: null } + ]) + expect(warn).toHaveBeenCalledWith( + '[journal-append] state row skipped:', + expect.objectContaining({ sessionId: 'settled' }) + ) + + db().exec('DROP TRIGGER fail_state_write') + await journal.close() + const reopened = await open('settled') + reopened.ensureSessionState() + expect(stored('settled')).toEqual(freshDerivation('settled')) + expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).not.toBeNull() + }) +}) + +describe('trusted only at the live tip (T9)', () => { + it('leaves the schema version alone, so an older build stays writable', async () => { + await write('settled') + expect(journalPragmaNumber(db(), 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) + expect(JOURNAL_DB_SCHEMA_VERSION).toBe(3) + }) + + it("reads an older build's append as stale, and the next open re-derives it", async () => { + const journal = await write('settled') + const tip = journal.cursor() + await journal.close() + // What an older build's append leaves: the row, and no state beside it. + insertTestJournalRow(db(), 'settled', { + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + kind: 'item', + itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'older-build-note' }), + revision: 1, + body: { kind: 'status', text: 'from an older build' }, + epoch: tip.epoch, + seq: tip.sequence + 1, + fence: 3, + ts: 5_000 + }) + expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toBeNull() + + const reopened = await open('settled') + reopened.ensureSessionState() + expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toMatchObject({ + seq: tip.sequence + 1, + lastActivityAt: 5_000 + }) + }) + + it("reads an older build's repair back to the same sequence as stale", async () => { + const journal = await write('settled') + const tip = journal.cursor() + const row = readJournalSessionState(db(), 'settled')! + // An older build repaired the chat after this row was written, then appended back to `seq`. + db() + .prepare( + 'INSERT INTO journal_repairs (session_id, epoch, content_from, repaired_at) VALUES (?, ?, ?, ?)' + ) + .run('settled', tip.epoch, tip.sequence, row.writtenAt + 1) + expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toBeNull() + + journal.ensureSessionState() + expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).not.toBeNull() + }) +}) + +describe('the epoch and history transactions keep it (T10)', () => { + it('writes it for a new epoch, a roll and a replacement, inside their transactions', async () => { + const journal = await open('epochs') + expect(stored('epochs')).toMatchObject({ + epoch: journal.epoch, + seq: 1, + owesWork: false, + summary: { status: null, latestPrompt: '' } + }) + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + expect(stored('epochs')).toMatchObject({ owesWork: true, summary: null }) + + await journal.rollEpoch('unreconcilable_prefix', 3) + expect(stored('epochs')).toEqual(freshDerivation('epochs')) + expect(stored('epochs')).toMatchObject({ epoch: journal.epoch, seq: 1, owesWork: false }) + + await journal.replaceEpochItems('legacy_import', 3, [ + { + identity: { provider: 'orca', clientMessageId: 'rebuilt-1' }, + body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'rebuilt' }] } + } + ]) + expect(stored('epochs')).toEqual(freshDerivation('epochs')) + expect(stored('epochs')).toMatchObject({ epoch: journal.epoch, seq: 2 }) + }) + + it('drops it with a repair, and the next open writes it again', async () => { + const journal = await write('settled') + const tip = journal.cursor() + await journal.close() + deleteJournalRepairedSuffix({ + database: openTestJournalHostDatabase(root), + sessionId: 'settled', + epoch: tip.epoch, + fromSeq: tip.sequence, + contentFrom: tip.sequence, + now: clock + 1 + }) + expect(readJournalSessionState(db(), 'settled')).toBeNull() + + const reopened = await open('settled') + reopened.ensureSessionState() + expect(stored('settled')).toEqual(freshDerivation('settled')) + }) +}) + +describe('the derivation is versioned (T11)', () => { + // Pinned with the version: a change to what a row holds must bump the version, so rows an + // earlier derivation wrote read as absent and are derived again. + const GOLDEN = { + version: 1, + digest: '637041b814cfbd16b3d5bd1b0e19f505bba360a3cc2d90f4ad801433b5192346' + } + + it('matches the digest pinned beside its version', async () => { + const derived: Record = {} + for (const name of JOURNAL_SESSION_STATE_CASES) { + await write(name) + const { epoch: _epoch, ...row } = freshDerivation(name) + derived[name] = row + } + const digest = createHash('sha256').update(JSON.stringify(derived)).digest('hex') + expect({ version: JOURNAL_SESSION_STATE_VERSION, digest }).toEqual(GOLDEN) + }) + + it('reads a row of another version as absent', async () => { + await write('settled') + db().prepare('UPDATE journal_session_state SET state_version = 99').run() + expect(readJournalSessionState(db(), 'settled')).toBeNull() + expect(readJournalSessionStatesAtTip(db(), ['settled'])).toEqual([ + { sessionId: 'settled', current: null } + ]) + }) +}) + +describe('a stored summary is fence-independent (T15a, regression guard)', () => { + it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { + await write(name) + const derived = freshDerivation(name) + if (derived.owesWork) { + expect(derived.summary).toBeNull() + return + } + const snapshot = renderJournalState(loadTestJournal(root, name)!.state) + for (const fence of [undefined, 0, 3, 4, 100]) { + expect( + projectStructuredAgentSessionStatusState(snapshot.items, snapshot.submissions, fence) + .summary + ).toEqual(derived.summary) + } + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts new file mode 100644 index 000000000000..819918c082e6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -0,0 +1,140 @@ +// The stored "owes work" flag and the open's settlement plan are two readings of one set of rules: +// the flag selects a chat exactly when its open would write something, the open writes exactly the +// plan, and once the plan commits the flag reads false again, so nothing is selected twice. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { + createTrackedJournalOpener, + liveTestJournalRows, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { owesOnOpen } from '../agent-session-journal/journal-open-settlement-plan' +import { readJournalSessionState } from '../agent-session-journal/journal-session-state' +import { + CORPUS_DEATH_EVIDENCE, + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS +} from '../agent-session-journal/journal-session-state-test-corpus' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + appendOpenSettlement, + openSettlementPlanIsEmpty, + planOpenSettlement, + type OpenSettlementRecordFacts +} from './structured-agent-session-open-settlement' + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 +let chats = 0 + +function open(sessionId: string): Promise { + const identity: AgentSessionJournalIdentity = { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + } + return journals.open({ + identity, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}` + }) +} + +function storedFacts(sessionId: string) { + const row = readJournalSessionState(openTestJournalHostDatabase(root).db, sessionId) + if (!row) { + throw new Error(`no stored state for ${sessionId}`) + } + return row +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-open-settlement-')) + clock = 1_000 +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +const COMBINATIONS = JOURNAL_SESSION_STATE_CASES.flatMap((name) => + Object.keys(CORPUS_DEATH_EVIDENCE).map((evidence) => [name, evidence] as const) +) + +describe('the stored flag and the plan agree (T4)', () => { + it.each(COMBINATIONS)('%s, death evidence %s', async (name, evidenceName) => { + const sessionId = `chat-${(chats += 1)}` + const journal = await open(sessionId) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + const deathEvidence = CORPUS_DEATH_EVIDENCE[evidenceName] ?? null + const record: OpenSettlementRecordFacts = { sessionId, fence: CORPUS_FENCE, deathEvidence } + + const plan = planOpenSettlement(journal, record, { settlesRosters: true }) + const selected = owesOnOpen(storedFacts(sessionId), deathEvidence) + expect(selected).toBe(!openSettlementPlanIsEmpty(plan)) + + // The open appends exactly the plan: a row per roster, per recovered send and per rejected + // leftover, and one lifecycle batch for what the gone generation left. + const before = liveTestJournalRows(openTestJournalHostDatabase(root).db, sessionId).length + await appendOpenSettlement(journal, plan, CORPUS_FENCE, (error) => { + throw error + }) + const appended = + liveTestJournalRows(openTestJournalHostDatabase(root).db, sessionId).length - before + expect(appended).toBe( + plan.rosters.length + + plan.recoveredDispatches.length + + plan.leftoverQueued.length + + ((plan.goneGeneration?.mutations.length ?? 0) > 0 ? 1 : 0) + ) + + // Every entry revised its entity out of the state that selected it (T4b, T15b). + expect(owesOnOpen(storedFacts(sessionId), deathEvidence)).toBe(false) + expect( + openSettlementPlanIsEmpty(planOpenSettlement(journal, record, { settlesRosters: true })) + ).toBe(true) + }) + + it('owes nothing for a settled chat, whatever the record says', async () => { + const journal = await open('settled') + await JOURNAL_SESSION_STATE_CORPUS.settled(journal) + for (const deathEvidence of Object.values(CORPUS_DEATH_EVIDENCE)) { + expect(owesOnOpen(storedFacts('settled'), deathEvidence)).toBe(false) + } + }) + + it("selects an unverifiable turn only by evidence naming its writer, never an older build's", async () => { + const journal = await open('unverifiable') + await JOURNAL_SESSION_STATE_CORPUS['unverifiable turn'](journal) + const facts = storedFacts('unverifiable') + expect(facts).toMatchObject({ owesWork: false, unverifiableOwnerFences: [CORPUS_FENCE] }) + expect(owesOnOpen(facts, CORPUS_DEATH_EVIDENCE['older build, no owner'])).toBe(false) + expect(owesOnOpen(facts, CORPUS_DEATH_EVIDENCE['names another owner'])).toBe(false) + expect(owesOnOpen(facts, CORPUS_DEATH_EVIDENCE['names the writer'])).toBe(true) + }) + + it('makes a chat whose only debt is a queued leftover owe work, and settles it (T15b)', async () => { + const journal = await open('queued') + await JOURNAL_SESSION_STATE_CORPUS['queued leftover'](journal) + expect(storedFacts('queued')).toMatchObject({ owesWork: true, summary: null }) + const plan = planOpenSettlement(journal, null, { settlesRosters: true }) + expect(plan.leftoverQueued).toEqual(['send-queued']) + + await appendOpenSettlement(journal, plan, CORPUS_FENCE, (error) => { + throw error + }) + + expect(journal.submission('send-queued')).toMatchObject({ dispatchState: 'rejected' }) + expect(storedFacts('queued')).toMatchObject({ owesWork: false, summary: { status: 'idle' } }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts new file mode 100644 index 000000000000..7eb5d8543632 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -0,0 +1,315 @@ +// Host startup from each chat's stored state: settled chats get their status rows without being +// opened, every chat that owes work is settled with no user action (tab or no tab), and a chat is +// opened only when stored state cannot answer for it. + +import { cp, mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION } from '../../../shared/agent-session-journal-types' +import { + closeTestJournalHostDatabases, + insertTestJournalRow, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { readJournalSessionState } from '../agent-session-journal/journal-session-state' +import Database from '../../sqlite/sync-database' +import { + createRestTestRig, + latestRestTestStatus, + restTestChat, + restTestOpens, + type RestTestRig +} from './structured-agent-session-rest-test-rig' + +const rigs: RestTestRig[] = [] + +afterEach(async () => { + for (const rig of rigs.splice(0)) { + await rig.dispose() + } + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +async function newRig(root?: string): Promise { + const rig = await createRestTestRig({}, root ? { root } : {}) + rigs.push(rig) + return rig +} + +function db(rig: RestTestRig) { + return openTestJournalHostDatabase(rig.root).db +} + +/** A send the provider only admitted, so it stays unanswered: the crash leaves it owed. */ +async function crashMidSend(rig: RestTestRig, sessionId: string, listed = true): Promise { + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) +} + +/** What startup runs, in order, on the ids the tab list names. */ +async function startup(rig: RestTestRig, listed: readonly string[]) { + await rig.host.reconcileRestartLeases() + const background = rig.host.seedStoredStatuses(listed) + await rig.host.settleOwedSessions(listed) + await rig.host.restoreReadableSessions(background) + return background +} + +function opened(rig: RestTestRig, ids: readonly string[]): string[] { + return ids.filter((sessionId) => restTestOpens(rig, sessionId) > 0) +} + +function statusRows(rig: RestTestRig, sessionId: string): AgentSessionStatusSummary[] { + return rig.statusEvents.flatMap((event) => + event.type === 'status' && event.session.sessionId === sessionId ? [event.session] : [] + ) +} + +const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds + +describe('seeding statuses from stored state', () => { + it('seeds each settled chat exactly as its open would publish it (T5)', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-answered', { message: 'finished work' }) + await restTestChat(rig, 'session-quiet') + rig.adapter.dispatch.mockResolvedValueOnce({ + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('providerRejected'), { + surface: 'rejection' + }) + }) + await restTestChat(rig, 'session-refused', { message: 'refused' }) + await rig.host.flushAllStreamedEvents() + const ids = ['session-answered', 'session-quiet', 'session-refused'] + closeTestJournalHostDatabases() + const readRoot = `${rig.root}-read` + await cp(rig.root, readRoot, { recursive: true }) + const readRig = await newRig(readRoot) + + const host = await rig.boot() + await host.reconcileRestartLeases() + expect(host.seedStoredStatuses(ids)).toEqual([]) + expect(opened(rig, ids)).toEqual([]) + // The same chats in another run, each opened by a read. + await readRig.host.reconcileRestartLeases() + for (const sessionId of ids) { + await readRig.host.history({ sessionId, direction: 'tail' }) + } + + for (const sessionId of ids) { + // Every field, `updatedAt` included: both come from the journal's own newest activity. + expect(latestRestTestStatus(rig, sessionId)).toEqual(latestRestTestStatus(readRig, sessionId)) + } + expect(latestRestTestStatus(rig, 'session-refused')).toMatchObject({ turnOutcome: 'failure' }) + // Opening a seeded chat finds its row equal and sends nothing. + const seeded = ids.map((sessionId) => statusRows(rig, sessionId).length) + for (const sessionId of ids) { + await host.history({ sessionId, direction: 'tail' }) + } + expect(ids.map((sessionId) => statusRows(rig, sessionId).length)).toEqual(seeded) + }) + + it('has every settled chat in the first snapshot a later subscriber gets (T8, T17)', async () => { + const rig = await newRig() + for (const sessionId of ['session-1', 'session-2']) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + const host = await rig.boot() + host.seedStoredStatuses(listedIds(rig)) + const snapshots: AgentSessionStatusSummary[][] = [] + host.subscribeStatus({ + id: 'remote-client', + emit: (event) => { + if (event.type === 'snapshot') { + snapshots.push(event.sessions) + } + } + }) + expect(snapshots[0]?.map((row) => [row.sessionId, row.status])).toEqual([ + ['session-1', 'idle'], + ['session-2', 'idle'] + ]) + }) + + it('reaches the status observers once per seeded chat, as a replay (T13)', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-1', { message: 'hi' }) + await rig.crash() + const onSessionStatusChanged = vi.fn() + const host = await rig.boot({ onSessionStatusChanged }) + host.seedStoredStatuses(['session-1']) + expect(onSessionStatusChanged).toHaveBeenCalledOnce() + expect(onSessionStatusChanged).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'session-1', status: 'idle' }), + { replay: true } + ) + }) + + it('drops a seeded chat from the status store when its tab closes (T12)', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-1', { message: 'hi' }) + await rig.crash() + const host = await rig.boot() + host.seedStoredStatuses(['session-1']) + expect(rig.sink.publish).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'session-1' }), + expect.anything() + ) + + await host.setSessionTabVisibility('session-1', false) + + expect(host.hasSession('session-1')).toBe(false) + expect(rig.sink.forget).toHaveBeenCalledOnce() + }) + + it('seeds nothing and settles nothing from a newer build database (T16, regression guard)', async () => { + const rig = await newRig() + await crashMidSend(rig, 'session-crashed') + await restTestChat(rig, 'session-settled', { message: 'done' }) + await rig.crash() + closeTestJournalHostDatabases() + const raw = new Database(join(rig.root, 'agent-session-journal.db')) + raw.pragma('user_version = 99') + raw.close() + const host = await rig.boot() + const ids = listedIds(rig) + + expect(host.seedStoredStatuses(ids)).toEqual(ids) + await host.settleOwedSessions(ids) + expect(rig.sink.publish).not.toHaveBeenCalled() + expect(opened(rig, ids)).toEqual([]) + }) +}) + +describe('startup opens only what it must (T6, T7, T14)', () => { + it('settles crashed chats with or without a tab, and opens only what stored state cannot answer', async () => { + const rig = await newRig() + const settled = Array.from({ length: 8 }, (_, index) => `session-settled-${index}`) + for (const sessionId of settled) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await crashMidSend(rig, 'session-crashed') + await crashMidSend(rig, 'session-crashed-closed', false) + for (const sessionId of [ + 'session-stale', + 'session-repaired', + 'session-draft', + 'session-paused', + 'session-uncopied' + ]) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + + const live = (sessionId: string) => + String( + db(rig).prepare('SELECT epoch FROM journal_sessions WHERE session_id = ?').get(sessionId) + ?.epoch + ) + const tip = (sessionId: string) => readJournalSessionState(db(rig), sessionId)! + // An older build appended past the stored state. + insertTestJournalRow(db(rig), 'session-stale', { + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + kind: 'item', + itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'older-build-note' }), + revision: 1, + body: { kind: 'status', text: 'from an older build' }, + epoch: live('session-stale'), + seq: tip('session-stale').seq + 1, + fence: 1, + ts: 9_000 + }) + // An older build repaired the chat after its state was stored, and wrote back to that sequence. + db(rig) + .prepare( + 'INSERT INTO journal_repairs (session_id, epoch, content_from, repaired_at) VALUES (?, ?, ?, ?)' + ) + .run( + 'session-repaired', + live('session-repaired'), + tip('session-repaired').seq, + tip('session-repaired').writtenAt + 1 + ) + const draft = db(rig).prepare( + `INSERT INTO queued_messages (session_id, message_id, position, body_json, fingerprint, + created_at, host_instance, state) VALUES (?, 'draft-1', 1, ?, 'fp', 1, 'host-a', 'waiting')` + ) + const body = JSON.stringify({ + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: 'x' }] + }) + draft.run('session-draft', body) + draft.run('session-paused', body) + db(rig) + .prepare( + `INSERT INTO queued_message_pauses (session_id, reason, epoch, sequence, recorded_at) + VALUES ('session-paused', 'stopped', ?, 1, 1)` + ) + .run(live('session-paused')) + // Still in the format a pre-database build wrote: nothing of it is in the host's database. + for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { + db(rig).prepare(`DELETE FROM ${table} WHERE session_id = ?`).run('session-uncopied') + } + const legacy = openTestJournalHostDatabase(rig.root).legacyDirectoryFor({ + workspaceId: 'workspace-1', + sessionId: 'session-uncopied' + }) + await mkdir(legacy, { recursive: true }) + await writeFile(join(legacy, 'log.jsonl'), '{}\n') + + await rig.boot() + const listed = listedIds(rig) + expect(listed).not.toContain('session-crashed-closed') + const background = await startup(rig, listed) + + expect(opened(rig, [...listed, 'session-crashed-closed']).toSorted()).toEqual( + [ + 'session-crashed', + 'session-crashed-closed', + 'session-draft', + 'session-repaired', + 'session-stale', + 'session-uncopied' + ].toSorted() + ) + expect(background.toSorted()).toEqual( + ['session-draft', 'session-repaired', 'session-stale', 'session-uncopied'].toSorted() + ) + // Settled with no user action; the listed one is open and never showed its pre-crash work. + for (const sessionId of ['session-crashed', 'session-crashed-closed']) { + // The unanswered send is now recovered doubt, which projects as no running request. + expect(readJournalSessionState(db(rig), sessionId)).toMatchObject({ + owesWork: false, + summary: { status: null } + }) + } + expect(rig.host.hasSession('session-crashed')).toBe(true) + expect(statusRows(rig, 'session-crashed').map((row) => row.status)).not.toContain('working') + // The tabless one is settled and closed: never indexed, never given a status row (T14). + expect(rig.host.hasSession('session-crashed-closed')).toBe(false) + expect( + rig.sink.publish.mock.calls.filter(([row]) => row.sessionId === 'session-crashed-closed') + ).toEqual([]) + // Every settled chat has its row without being opened. + for (const sessionId of settled) { + expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) + } + + // The next boot finds nothing owed and nothing stale (T4b). + await rig.crash() + await rig.boot() + const again = await startup(rig, listedIds(rig)) + expect(opened(rig, [...listedIds(rig), 'session-crashed-closed']).toSorted()).toEqual( + again.toSorted() + ) + // Only the chat whose draft still waits is opened again, as every listed chat used to be. + expect(again).toEqual(['session-draft']) + }) +}) diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts new file mode 100644 index 000000000000..1d9eb95b5b85 --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -0,0 +1,121 @@ +// The host's own startup step, run from `prepare` whether or not any client ever lists a tab: a +// headless host has its seeded statuses and its settled crashed chats, and no failure in the step +// (the lease check, one chat's open, one chat's settlement) costs startup or the other chats. + +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { closeTestJournalHostDatabases } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { readJournalSessionState } from '../native-chat/agent-session-journal/journal-session-state' +import { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' +import { openTestJournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import { + createRestTestRig, + latestRestTestStatus, + restTestChat, + restTestOpens, + type RestTestRig +} from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' +import { OrcaRuntimeService } from './orca-runtime' + +type PrepareInternals = { + store: { getWorkspaceSession: () => unknown } + hasPersistedStructuredAgentSessionStore(): boolean + refreshMobileSessionPtyRecords(): Promise | null> + ensureStructuredAgentSessionHost(): Promise +} + +let rig: RestTestRig + +beforeEach(async () => { + rig = await createRestTestRig() +}) + +afterEach(async () => { + setStructuredAgentSessionHost(null) + await rig.dispose() + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +/** A restarted runtime over the rig's host, with nothing but `prepare` ever called on it. */ +function restartedRuntime(): OrcaRuntimeService { + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + const internal = runtime as unknown as PrepareInternals + internal.store = { getWorkspaceSession: () => null } + internal.hasPersistedStructuredAgentSessionStore = () => true + internal.refreshMobileSessionPtyRecords = async () => new Set() + internal.ensureStructuredAgentSessionHost = async () => { + setStructuredAgentSessionHost(rig.host) + } + return runtime +} + +async function crashMidSend(sessionId: string, listed = true): Promise { + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) +} + +function owes(sessionId: string): boolean | undefined { + return readJournalSessionState(openTestJournalHostDatabase(rig.root).db, sessionId)?.owesWork +} + +it('seeds and settles on a host no client ever lists (T8)', async () => { + await restTestChat(rig, 'session-settled', { message: 'done' }) + await crashMidSend('session-crashed') + await crashMidSend('session-crashed-closed', false) + await rig.crash() + await rig.boot() + const runtime = restartedRuntime() + + await runtime.prepareStructuredAgentSessionStartupRestoration() + + expect(latestRestTestStatus(rig, 'session-settled')).toMatchObject({ status: 'idle' }) + await vi.waitFor(() => { + expect(owes('session-crashed')).toBe(false) + expect(owes('session-crashed-closed')).toBe(false) + }) + expect(rig.host.hasSession('session-crashed')).toBe(true) + expect(rig.host.hasSession('session-crashed-closed')).toBe(false) + expect(restTestOpens(rig, 'session-settled')).toBe(0) +}) + +it('never fails startup: each failure is logged by chat and the rest still settle (T18)', async () => { + await restTestChat(rig, 'session-settled', { message: 'done' }) + for (const sessionId of ['session-open-fails', 'session-settle-fails', 'session-fine']) { + await crashMidSend(sessionId) + } + await rig.crash() + await rig.boot() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const leaseCheck = new Error('lease check failed') + vi.spyOn(rig.host, 'reconcileRestartLeases').mockRejectedValueOnce(leaseCheck) + rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { + if (sessionId === 'session-open-fails') { + throw new Error('EACCES: permission denied') + } + return null + }) + const markUnknown = AgentSessionJournal.prototype.markPendingSubmissionsUnknown + vi.spyOn(AgentSessionJournal.prototype, 'markPendingSubmissionsUnknown').mockImplementation( + function (this: AgentSessionJournal, ...args) { + if (this.snapshot().sessionId === 'session-settle-fails') { + return Promise.reject(new Error('settlement append failed')) + } + return markUnknown.apply(this, args) + } + ) + const runtime = restartedRuntime() + + // The lease check's failure is still the step's answer, unchanged. + await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).rejects.toBe(leaseCheck) + + expect(latestRestTestStatus(rig, 'session-settled')).toMatchObject({ status: 'idle' }) + await vi.waitFor(() => expect(owes('session-fine')).toBe(false)) + expect(owes('session-open-fails')).toBe(true) + expect(owes('session-settle-fails')).toBe(true) + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] restoring a chat for reading failed', + expect.objectContaining({ sessionId: 'session-open-fails' }) + ) +}) From bf0796ed38a875746d03e3d76c64439651b1140f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:57:59 -0700 Subject: [PATCH 020/107] test(native-chat): one import of the journal test support in the startup restoration test --- .../structured-agent-session-startup-restoration.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index 1d9eb95b5b85..692dbe67d8b2 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -3,10 +3,12 @@ // (the lease check, one chat's open, one chat's settlement) costs startup or the other chats. import { afterEach, beforeEach, expect, it, vi } from 'vitest' -import { closeTestJournalHostDatabases } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../native-chat/agent-session-journal/journal-host-database-test-support' import { readJournalSessionState } from '../native-chat/agent-session-journal/journal-session-state' import { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' -import { openTestJournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database-test-support' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { createRestTestRig, From b088396b3fee91d86555f4cb657c40680e6da617 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:30:31 -0700 Subject: [PATCH 021/107] test(native-chat): settled roster uses a real terminal state; pin what each corpus case owes The corpus's finished roster used "done", which no subagent state is: it read as unverifiable only through the unknown-value fallback. It is "completed" now, and each case's stored flag is pinned, so the flag and the plan agreeing cannot hide both being wrong. --- .../journal-session-state-test-corpus.ts | 21 +++++++++++++++++-- ...ured-agent-session-open-settlement.test.ts | 3 +++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts index c0ff4d627095..530db21d51fb 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts @@ -67,7 +67,7 @@ async function settledTurn(journal: AgentSessionJournal, turnId: string): Promis }) } -function roster(state: 'working' | 'done'): AgentJournalItemBody { +function roster(state: 'working' | 'completed'): AgentJournalItemBody { return { kind: 'message', role: 'system', @@ -167,12 +167,29 @@ export const JOURNAL_SESSION_STATE_CORPUS = { }, 'settled roster': async (journal: AgentSessionJournal) => { await settledTurn(journal, 'turn-1') - await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('done')) + await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('completed')) } } satisfies Record Promise> export type JournalSessionStateCase = keyof typeof JOURNAL_SESSION_STATE_CORPUS +/** Whether each case owes its open work whatever the record says: what the flag must read. */ +export const CORPUS_OWES_WORK: Record = { + empty: false, + settled: false, + 'older turn running beside a newer one': true, + 'working subagent roster': true, + 'live background task': true, + 'pending prompt': true, + 'running tool': true, + 'pending send': true, + 'unknown send': true, + 'queued leftover': true, + // Owed only to death evidence naming its writer. + 'unverifiable turn': false, + 'settled roster': false +} + export const JOURNAL_SESSION_STATE_CASES = Object.keys(JOURNAL_SESSION_STATE_CORPUS).filter( (name): name is JournalSessionStateCase => name in JOURNAL_SESSION_STATE_CORPUS ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts index 819918c082e6..04f92fd7bdc7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -17,6 +17,7 @@ import { readJournalSessionState } from '../agent-session-journal/journal-sessio import { CORPUS_DEATH_EVIDENCE, CORPUS_FENCE, + CORPUS_OWES_WORK, JOURNAL_SESSION_STATE_CASES, JOURNAL_SESSION_STATE_CORPUS } from '../agent-session-journal/journal-session-state-test-corpus' @@ -79,6 +80,8 @@ describe('the stored flag and the plan agree (T4)', () => { const deathEvidence = CORPUS_DEATH_EVIDENCE[evidenceName] ?? null const record: OpenSettlementRecordFacts = { sessionId, fence: CORPUS_FENCE, deathEvidence } + // Pinned per case, so agreement between the flag and the plan cannot hide both being wrong. + expect(storedFacts(sessionId).owesWork).toBe(CORPUS_OWES_WORK[name]) const plan = planOpenSettlement(journal, record, { settlesRosters: true }) const selected = owesOnOpen(storedFacts(sessionId), deathEvidence) expect(selected).toBe(!openSettlementPlanIsEmpty(plan)) From 617b30ef327de63c109000bfa18bfa574eb18fd9 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:57:06 -0700 Subject: [PATCH 022/107] test(native-chat): close the chat as the user does in the restore-after-listing test Main's host close now takes the cause, and a closed tab is the user's close. --- .../structured-agent-session-restore-after-listing.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts index 09ac84c75f5d..48736a8c7ac8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts @@ -57,7 +57,7 @@ describe('the restore pass after the tab list', () => { // What closing the tab does. await host.setSessionTabVisibility(LAST, false) - await host.close(LAST) + await host.close(LAST, 'user-close') const publishedAtClose = publishedFor(LAST) release() await pass From f52521dd9bb91d0896308700e80057bd23c4e913 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:13:40 -0700 Subject: [PATCH 023/107] refactor(native-chat): store each chat's state after its journal commit The append folds after COMMIT again, as before this branch, and the chat's stored state is written right after in its own unsynced transaction. A failed or lost state write is logged and reads as stale by position, so the next open or startup re-derives it. This removes the fold inside the transaction, the savepoint, the re-fold from disk and the stranded close. Epoch start, roll and replacement store their state through the same after-commit write, so a state failure no longer fails them. Also: the stored flag follows the current load (a chat that writes past a repair, or is rebuilt, owes its rosters again); items whose key will not parse owe nothing, as no plan can revise them; the status feed and the stored state share one projection per commit; startup no longer opens chats for a waiting draft, which a restart pauses; an unknown stored verdict reads as stale. --- .../first-work-branch-rename.test.ts | 48 +++--- .../journal-append-fold-recovery.test.ts | 160 ------------------ .../journal-epoch-controller.ts | 6 +- .../journal-epoch-replacement.test.ts | 1 - .../journal-epoch-replacement.ts | 8 +- .../journal-epoch-rollover.ts | 34 ++-- .../journal-host-database.ts | 5 - .../journal-open-settlement-plan.ts | 32 ++-- .../journal-row-writer.test.ts | 6 +- .../journal-row-writer.ts | 75 ++------ .../journal-session-state.test.ts | 141 +++++++++++++-- .../journal-session-state.ts | 62 ++++--- .../journal-status-projection.ts | 59 +++++++ .../journal-store-collaborators.ts | 134 ++++++--------- .../journal-store-restore.ts | 4 +- .../agent-session-journal/journal-store.ts | 61 ++++--- .../structured-agent-session-conversations.ts | 8 - ...ured-agent-session-open-settlement.test.ts | 90 +++++++++- ...tructured-agent-session-open-settlement.ts | 21 ++- ...ctured-agent-session-startup-state.test.ts | 77 ++++----- .../structured-agent-session-startup-state.ts | 13 +- ...ructured-agent-session-status-feed.test.ts | 6 +- .../structured-agent-session-status-feed.ts | 56 ++---- src/main/orcad/orcad-push-startup.test.ts | 1 + ...ructured-agent-session-rpc.test-fixture.ts | 8 +- 25 files changed, 558 insertions(+), 558 deletions(-) delete mode 100644 src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-status-projection.ts diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index 63b6165ce3aa..66ccc8ba62df 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { GlobalSettings } from '../../shared/global-settings-types' import type { Repo } from '../../shared/repo-types' import type { AgentJournalRenderItem } from '../../shared/agent-session-journal-types' -import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' +import { projectStructuredAgentSessionStatusState } from '../../shared/structured-agent-session-projection' import { StructuredAgentSessionStatusFeed } from '../native-chat/agent-session-wire/structured-agent-session-status-feed' import { maybeAutoRenameWorkspaceOnFirstStructuredTurn } from './first-work-structured-session-rename' import { WORKTREE_ID_SEPARATOR } from '../../shared/worktree/id' @@ -100,15 +100,13 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { isPendingFirstAgentMessageRename: () => true }) const items: AgentJournalRenderItem[] = [] - // A real journal's sequence only ever advances, so the feed's projection - // cache must miss on every publish here: this test is about the rename. - let sequence = 0 + // Projected on every publish, as a real journal re-projects each commit: this test is + // about the rename. const journal = { - snapshot: () => ({ items }), lastActivityAt: () => 1, isReadOnly: false, - cursor: () => ({ epoch: 1, sequence: (sequence += 1) }) - } as unknown as AgentSessionJournal + statusState: (fence?: number) => projectStructuredAgentSessionStatusState(items, [], fence) + } const pending: Promise[] = [] const observe = vi.fn((summary, options) => { const work = maybeAutoRenameWorkspaceOnFirstStructuredTurn(summary, options, deps) @@ -190,23 +188,31 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { const { deps, setDisplayName, setRenameError } = makeDeps({ getRepo: () => ({ id: REPO_ID, kind: 'folder', path: '/workspace/platform' }) as Repo }) + const items: AgentJournalRenderItem[] = [ + { + itemId: 'user-1', + sequence: 1, + revision: 1, + observedAt: 1, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Fix auth' }] } + }, + { + itemId: 'turn-1', + sequence: 2, + revision: 1, + observedAt: 1, + body: { + kind: 'status', + text: 'Working', + turnLifecycle: { turnId: 'turn-1', state: 'running' } + } + } + ] const journal = { isReadOnly: false, lastActivityAt: () => 1, - cursor: () => ({ epoch: 1, sequence: 1 }), - snapshot: () => ({ - items: [ - { body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Fix auth' }] } }, - { - body: { - kind: 'status', - text: 'Working', - turnLifecycle: { turnId: 'turn-1', state: 'running' } - } - } - ] - }) - } as unknown as AgentSessionJournal + statusState: (fence?: number) => projectStructuredAgentSessionStatusState(items, [], fence) + } const location = { executionHostId: 'local' as const, wslDistro: null, diff --git a/src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts b/src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts deleted file mode 100644 index e64345417028..000000000000 --- a/src/main/native-chat/agent-session-journal/journal-append-fold-recovery.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -// An append folds its row inside the transaction, so the chat's stored state can describe it. A -// transaction that fails after that fold leaves the fold ahead of the disk: the chat re-folds from -// what committed, or closes when a stranded connection could still show it the uncommitted row. - -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' -import { - createTrackedJournalOpener, - loadTestJournal, - openTestJournalHostDatabase -} from './journal-host-database-test-support' -import { applyJournalRow, renderJournalState } from './journal-reducer' -import type * as JournalReducer from './journal-reducer' -import { readJournalSessionState } from './journal-session-state' -import type { AgentSessionJournal } from './journal-store' - -vi.mock('./journal-reducer', async (importOriginal) => { - const actual = await importOriginal() - return { ...actual, applyJournalRow: vi.fn(actual.applyJournalRow) } -}) - -const SESSION = 'session-fold' -const IDENTITY: AgentSessionJournalIdentity = { - sessionId: SESSION, - workspaceId: 'ws-1', - hostId: 'local', - agent: 'codex', - providerHandle: { kind: 'codex', threadId: 'thread-fold' } -} - -const journals = createTrackedJournalOpener() -let root: string -let clock = 1_000 - -function open(): Promise { - return journals.open({ - identity: IDENTITY, - stateDirectory: root, - now: () => (clock += 1), - mintEpoch: () => 'epoch-fold' - }) -} - -function note(journal: AgentSessionJournal, id: string) { - return journal.appendItem( - { provider: 'orca', clientMessageId: id }, - { kind: 'status', text: id }, - { fence: 1, turnScope: { kind: 'thread' } } - ) -} - -/** The chat's fold, as a reader of it sees it. */ -function foldOf(journal: AgentSessionJournal) { - return { - cursor: journal.cursor(), - snapshot: journal.snapshot(), - lastActivityAt: journal.lastActivityAt(), - highestFence: journal.highestFence() - } -} - -/** The same, from a fresh replay of what is on disk. */ -function durableFold() { - const loaded = loadTestJournal(root, SESSION)! - return { - cursor: { epoch: loaded.state.epoch, sequence: loaded.state.lastSequence }, - snapshot: renderJournalState(loaded.state), - lastActivityAt: loaded.state.lastActivityAt, - highestFence: loaded.state.highestFence - } -} - -/** Fails the next statement(s) the connection runs whose SQL is in `failing`, once each. */ -function failOnce(failing: string[]): void { - const database = openTestJournalHostDatabase(root) - const connection = database.db - const exec = connection.exec.bind(connection) - const pending = new Set(failing) - vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { - if (pending.delete(sql)) { - throw new Error(`${sql} failed: disk I/O error`) - } - exec(sql) - }) -} - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'orca-journal-fold-recovery-')) - clock = 1_000 -}) - -afterEach(async () => { - vi.restoreAllMocks() - await journals.closeAll() - await rm(root, { recursive: true, force: true }) -}) - -describe('an append whose transaction fails after its row folded', () => { - it('re-folds from disk when the fold itself throws part-way (T1a)', async () => { - const journal = await open() - await note(journal, 'first') - const stateBefore = readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION) - // One mutation lands, then the fold throws: the in-place apply cannot be undone. - vi.mocked(applyJournalRow).mockImplementationOnce((state, row) => { - state.lastSequence = row.seq - throw new Error('fold failed part-way') - }) - - await expect(note(journal, 'second')).rejects.toThrow('fold failed part-way') - - expect(durableFold().cursor.sequence).toBe(2) - expect(foldOf(journal)).toEqual(durableFold()) - expect(readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION)).toEqual( - stateBefore - ) - // The next append takes the sequence the failed one could not. - await expect(note(journal, 'third')).resolves.toMatchObject({ cursor: { sequence: 3 } }) - }) - - it('re-folds from disk when COMMIT fails and the rollback goes through (T1b)', async () => { - const journal = await open() - await note(journal, 'first') - const stateBefore = readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - failOnce(['COMMIT']) - - await expect(note(journal, 'second')).rejects.toThrow('COMMIT failed') - - expect(durableFold().cursor.sequence).toBe(2) - expect(foldOf(journal)).toEqual(durableFold()) - expect(readJournalSessionState(openTestJournalHostDatabase(root).db, SESSION)).toEqual( - stateBefore - ) - await expect(note(journal, 'third')).resolves.toMatchObject({ cursor: { sequence: 3 } }) - }) - - it('closes the chat rather than re-reading a stranded connection (T1c)', async () => { - const journal = await open() - await note(journal, 'first') - const stranded = vi.fn() - journal.observeStranded(stranded) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - failOnce(['COMMIT', 'ROLLBACK']) - - await expect(note(journal, 'second')).rejects.toThrow('COMMIT failed') - - expect(stranded).toHaveBeenCalledOnce() - // Closed: no later write can land past a row that never committed. - await expect(note(journal, 'third')).rejects.toMatchObject({ code: 'journal_closed' }) - // The host database's own retry rolls the stranded transaction back on its next use. - expect(openTestJournalHostDatabase(root).db.isTransaction).toBe(false) - expect(durableFold().cursor.sequence).toBe(2) - const reopened = await open() - expect(foldOf(reopened)).toEqual(durableFold()) - await expect(note(reopened, 'fourth')).resolves.toMatchObject({ cursor: { sequence: 3 } }) - }) -}) diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index 6ce527e9eab4..52a5478c381a 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -4,7 +4,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import { replaceJournalEpoch, type JournalReplacementItem } from './journal-epoch-replacement' -import { publishNewEpoch, type JournalEpochStateWriter } from './journal-epoch-rollover' +import { publishNewEpoch } from './journal-epoch-rollover' import type { JournalLoad } from './journal-open' import type { AgentJournalEpochReason } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' @@ -22,8 +22,6 @@ export class JournalEpochController { highestFence: () => number cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void - /** The chat's stored state for the new epoch, inside its transaction. */ - writeState: JournalEpochStateWriter } ) {} @@ -35,7 +33,6 @@ export class JournalEpochController { reason, fence, now: this.deps.now(), - writeState: this.deps.writeState, onPublished: this.deps.adopt }) } @@ -72,7 +69,6 @@ export class JournalEpochController { items, now: this.deps.now, mintEpoch: this.deps.mintEpoch, - writeState: this.deps.writeState, onPublished: this.deps.adopt }) return this.deps.cursor() diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts index 63bccce5582d..646254b4932d 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts @@ -69,7 +69,6 @@ function replace(input: { items: input.items, now, mintEpoch: () => `epoch-${clock}`, - writeState: () => undefined, onPublished: input.onPublished ?? (() => undefined) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 1c4acbf7e957..8d94c5b2b5b0 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -1,9 +1,8 @@ // Republishing a live item set into a fresh epoch. // // One transaction: discard the old epoch's rows, insert the epoch row plus the -// replacement items, move the session projection, retire any repair marker -// — this republished history is exactly what the marker was holding out for — -// and store the chat's state for the new epoch. +// replacement items, move the session projection, and retire any repair marker +// — this republished history is exactly what the marker was holding out for. import type { AgentJournalItemBody, @@ -15,7 +14,6 @@ import type { import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' import { clearJournalRepairMarker } from './journal-repair-marker' -import type { JournalEpochStateWriter } from './journal-epoch-rollover' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { buildJournalItemRow, journalRowBase } from './journal-row-builders' import { @@ -44,7 +42,6 @@ export function replaceJournalEpoch(input: { items: readonly JournalReplacementItem[] now: () => number mintEpoch: () => string - writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -85,7 +82,6 @@ export function replaceJournalEpoch(input: { insertJournalRow(db, sessionId, row) } publishJournalSessionEpoch(db, input.identity, epoch) - input.writeState(db, state) }) // COMMIT landed: on disk the superseded rows are gone and this epoch is the diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts index afd6c0e3cb57..b7eb16b81a1d 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts @@ -1,22 +1,16 @@ // Opening a new epoch. // // One transaction: discard every row of the superseded epoch, insert the new -// epoch row at sequence 1, move the session projection onto it, retire any -// repair marker the superseded epoch was carrying, and store the chat's state for -// the new epoch. Superseded rows are DELETED rather than retained — nothing would -// ever shed them. +// epoch row at sequence 1, move the session projection onto it, and retire any +// repair marker the superseded epoch was carrying. Superseded rows are DELETED +// rather than retained — nothing would ever shed them. import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' import { clearJournalRepairMarker } from './journal-repair-marker' -import type Database from '../../sqlite/sync-database' -import { - applyJournalRow, - createJournalReducerState, - type JournalReducerState -} from './journal-reducer' +import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { deleteJournalEpochRows, insertJournalRow, @@ -25,9 +19,6 @@ import { } from './journal-row-table' import type { AgentJournalEpochReason, JournalRow } from './journal-row-schema' -/** Writes the chat's stored state from the fold an epoch transaction is about to publish. */ -export type JournalEpochStateWriter = (db: Database.Database, state: JournalReducerState) => void - export function publishNewEpoch(input: { database: JournalHostDatabase identity: AgentSessionJournalIdentity @@ -35,7 +26,6 @@ export function publishNewEpoch(input: { reason: AgentJournalEpochReason fence: number now: number - writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -52,10 +42,6 @@ export function publishNewEpoch(input: { } const { sessionId } = input.identity - // Folded first, so the transaction stores the state of the epoch it publishes. - const state = createJournalReducerState(sessionId, input.epoch) - applyJournalRow(state, row) - state.oldestSequence = 1 input.database.transaction((db) => { const retired = readJournalSessionEpoch(db, sessionId) if (retired !== null) { @@ -64,11 +50,19 @@ export function publishNewEpoch(input: { clearJournalRepairMarker(db, sessionId) insertJournalRow(db, sessionId, row) publishJournalSessionEpoch(db, input.identity, input.epoch) - input.writeState(db, state) }) // COMMIT landed: on disk the superseded prefix is gone and this epoch is the // live one. The caller adopts that immediately, or a later failure leaves the // store writing into an epoch that no longer exists. - input.onPublished({ state, readOnly: false, corrupt: false, malformedRows: 0 }) + const state = createJournalReducerState(sessionId, input.epoch) + applyJournalRow(state, row) + state.oldestSequence = 1 + // A repair's epoch reads back corrupt until the chat writes past it, as its replay would. + input.onPublished({ + state, + readOnly: false, + corrupt: input.reason === 'unreconcilable_prefix', + malformedRows: 0 + }) } diff --git a/src/main/native-chat/agent-session-journal/journal-host-database.ts b/src/main/native-chat/agent-session-journal/journal-host-database.ts index 2a85b6261a27..ca6036d3a9d0 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database.ts @@ -77,11 +77,6 @@ export class JournalHostDatabase { return this.connection === null } - /** A ROLLBACK failed: the connection may still hold rows that never committed. */ - get isStranded(): boolean { - return this.stranded - } - get db(): Database.Database { const connection = this.connection if (!connection) { diff --git a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts index 2b83b0a7ab1b..af363fbc3a3d 100644 --- a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts +++ b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts @@ -5,8 +5,10 @@ // owes work exactly when its open would write something. Each rule revises its entity out of the // state that selected it, so once the plan commits the flag reads false again. +import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { AgentJournalItemBody, + AgentJournalItemIdentity, AgentJournalRenderItem, AgentJournalSubmission } from '../../../shared/agent-session-journal-types' @@ -16,6 +18,14 @@ import { cancelledJournalPromptBody } from './journal-prompt-body-bounds' import type { JournalReducerState } from './journal-reducer' import { staleSubagentRosterRevision } from './journal-subagent-liveness' +/** The key a settlement revises an item under. One that will not parse cannot be revised, so the + * item owes nothing: a fresh identity would duplicate it rather than settle it. */ +export function openSettlementItemIdentity( + item: Pick +): AgentJournalItemIdentity | null { + return parseAgentJournalItemKey(item.itemId) +} + /** A running tool call fails, and a pending approval or question is cancelled. */ export function openSettlementTerminalBody( item: Pick @@ -43,11 +53,6 @@ export function owesRecoveredDispatch(submission: AgentJournalSubmission): boole ) } -/** A send accepted and never handed over is a leftover: nothing indexes a chat with one queued. */ -export function isLeftoverQueuedSubmission(submission: AgentJournalSubmission): boolean { - return isQueuedAgentJournalSubmission(submission) -} - /** The writer fence of an `unverifiable` turn, which only death evidence naming it revises. */ export function unverifiableTurnOwnerFence( item: Pick, @@ -73,7 +78,8 @@ export function owesOpenSettlement( ): JournalOwedFacts { let owesWork = false for (const submission of fold.submissions.values()) { - if (owesRecoveredDispatch(submission) || isLeftoverQueuedSubmission(submission)) { + // A queued one is a leftover: its process is gone, since a close abandons what it queued. + if (owesRecoveredDispatch(submission) || isQueuedAgentJournalSubmission(submission)) { owesWork = true break } @@ -81,14 +87,18 @@ export function owesOpenSettlement( const fences = new Set() const itemFence = (itemId: string) => fold.itemFences.get(itemId) for (const item of fold.items.values()) { + owesWork ||= options.settlesRosters && staleSubagentRosterRevision(item) !== null const fence = unverifiableTurnOwnerFence(item, itemFence) - if (fence !== undefined) { + const settles = + fence !== undefined || isRunningJournalTurn(item) || openSettlementTerminalBody(item) !== null + if (!settles || !openSettlementItemIdentity(item)) { + continue + } + if (fence === undefined) { + owesWork = true + } else { fences.add(fence) } - owesWork ||= - isRunningJournalTurn(item) || - openSettlementTerminalBody(item) !== null || - (options.settlesRosters && staleSubagentRosterRevision(item) !== null) } return { owesWork, unverifiableOwnerFences: [...fences].sort((a, b) => a - b) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts index 8569bdb14f49..aa0078b88a59 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts @@ -68,12 +68,10 @@ describe('journal row writer', () => { readOnly: () => readOnly, highestFence: () => 0, nextSequence: () => sequence, - apply: (committed) => { + commit: (committed) => { committedRows.push(committed) sequence = committed.seq + 1 - }, - committed: () => undefined, - recoverFold: () => undefined + } }) return { writer, committedRows } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index eef89bb363fa..a29585580ebe 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -18,25 +18,16 @@ export type JournalRowWriterDeps = { readOnly: () => boolean highestFence: () => number nextSequence: () => number - /** Folds the row, inside the transaction, so the state row below describes it. */ - apply: (row: JournalRow) => void - /** After COMMIT: observers learn of the row only once it is durable. */ - committed: () => void - /** A transaction that failed after `apply`: the fold is ahead of the disk. Re-folds from disk, - * or closes the chat when the connection is stranded and a re-read would see the row. */ - recoverFold: () => void + commit: (row: JournalRow) => void /** Standing hook run for EVERY appended row — the queued-draft returned * transition rides here so no rejection path can bypass it. Bookkeeping: it * runs in its own savepoint, so its failure is reported and never vetoes the row. */ inTransaction?: JournalRowTransactionHook - /** The chat's stored state, from the fold that now includes the row. Bookkeeping too. */ - writeState?: (db: Database.Database) => void /** After any rollback, so a cache filled inside the transaction cannot outlive it. */ rolledBack?: () => void } const BOOKKEEPING_SAVEPOINT = 'journal_row_bookkeeping' -const STATE_SAVEPOINT = 'journal_session_state' export class JournalRowWriter { constructor(private readonly deps: JournalRowWriterDeps) {} @@ -49,26 +40,21 @@ export class JournalRowWriter { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) const row = build(this.deps.nextSequence(), this.deps.now()) assertJournalFence(row.fence, this.deps.highestFence()) - let applied = false try { // One INSERT: the chat's epoch pointer moves only when the epoch does. this.deps.database().transaction((db) => { insertJournalRow(db, this.deps.sessionId, row) hook?.(db, row) this.runBookkeeping(db, row) - applied = true - this.deps.apply(row) - this.runStateWrite(db, row) }) } catch (error) { this.deps.rolledBack?.() - if (applied) { - this.deps.recoverFold() - } throw error } - // COMMIT landed and the fold already holds the row; only now do observers hear of it. - this.deps.committed() + // COMMIT landed, so the row is durable: adopt it before anything that can + // fail. Rejecting here instead would leave the next append reusing a + // sequence the table already holds. + this.deps.commit(row) return row }) } @@ -87,50 +73,17 @@ export class JournalRowWriter { if (!hook) { return } - this.inSavepoint( - db, - BOOKKEEPING_SAVEPOINT, - row, - () => hook(db, row), - () => { - this.deps.rolledBack?.() - // The draft store re-derives what this missed from the committed rows: at open, and in - // the drain step before a draft sends. - return 'row bookkeeping skipped' - } - ) - } - - private runStateWrite(db: Database.Database, row: JournalRow): void { - const write = this.deps.writeState - if (write) { - // Left behind the tip on failure, so the next open or startup re-derives it. - this.inSavepoint( - db, - STATE_SAVEPOINT, - row, - () => write(db), - () => 'state row skipped' - ) - } - } - - /** Bookkeeping never vetoes the row: its failure rolls back to the savepoint and is reported. */ - private inSavepoint( - db: Database.Database, - name: string, - row: JournalRow, - run: () => void, - onFailure: () => string - ): void { - db.exec(`SAVEPOINT ${name}`) + db.exec(`SAVEPOINT ${BOOKKEEPING_SAVEPOINT}`) try { - run() - db.exec(`RELEASE ${name}`) + hook(db, row) + db.exec(`RELEASE ${BOOKKEEPING_SAVEPOINT}`) } catch (error) { - db.exec(`ROLLBACK TO ${name}`) - db.exec(`RELEASE ${name}`) - console.warn(`[journal-append] ${onFailure()}:`, { + db.exec(`ROLLBACK TO ${BOOKKEEPING_SAVEPOINT}`) + db.exec(`RELEASE ${BOOKKEEPING_SAVEPOINT}`) + this.deps.rolledBack?.() + // The draft store re-derives what this missed from the committed rows: at open, and in + // the drain step before a draft sends. + console.warn('[journal-append] row bookkeeping skipped:', { sessionId: this.deps.sessionId, kind: row.kind, error: error instanceof Error ? error.message : String(error) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 275199631ba2..30dab49af63e 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -1,5 +1,5 @@ -// Each chat's stored state: written in the journal row's own transaction, equal to what a fresh -// replay derives, never able to fail an append, and trusted only at the chat's live tip. +// Each chat's stored state: written right after each journal commit, equal to what a fresh replay +// derives, never able to fail the write it describes, and trusted only at the chat's live tip. import { createHash } from 'node:crypto' import { isDeepStrictEqual } from 'node:util' @@ -18,6 +18,7 @@ import { journalPragmaNumber } from './journal-database' import { createTrackedJournalOpener, insertTestJournalRow, + insertTestJournalRowJson, loadTestJournal, openTestJournalHostDatabase } from './journal-host-database-test-support' @@ -103,12 +104,14 @@ describe('the stored state follows every append (T3)', () => { it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { const journal = await open(name) const mismatches: string[] = [] - // Told after each commit: the row and its state landed in one transaction. + // Observers hear of a commit first; its state lands in the same task, before the next write. journal.observeCommits(() => { - const expected = freshDerivation(name) - if (!isDeepStrictEqual(stored(name), expected)) { - mismatches.push(`seq ${expected.seq}`) - } + queueMicrotask(() => { + const expected = freshDerivation(name) + if (!isDeepStrictEqual(stored(name), expected)) { + mismatches.push(`seq ${expected.seq}`) + } + }) }) await JOURNAL_SESSION_STATE_CORPUS[name](journal) await Promise.resolve() @@ -117,13 +120,70 @@ describe('the stored state follows every append (T3)', () => { }) }) +function refuseStateWrites(): void { + db().exec(`CREATE TEMP TRIGGER fail_state_write BEFORE UPDATE ON main.journal_session_state + BEGIN SELECT RAISE(ABORT, 'state write refused'); END`) +} + +describe('a chat that opened corrupt stores what a fresh replay derives (T3)', () => { + const name = 'working subagent roster' + + async function openCorrupt(): Promise { + const journal = await write(name) + const tip = journal.cursor() + await journal.close() + // A bad write past the tip: the next open drops it and owes a rebuild from provider history. + insertTestJournalRowJson(db(), name, tip.sequence + 1, '{"not a row"') + const reopened = await open(name) + expect(reopened.needsRebuild).toBe(true) + // Rosters wait for the rebuild, as the open's plan does. + expect(stored(name)).toEqual(freshDerivation(name)) + expect(stored(name)).toMatchObject({ owesWork: false }) + return reopened + } + + it('owes the roster again once the chat writes past the repair', async () => { + const journal = await openCorrupt() + await journal.appendItem( + { provider: 'orca', clientMessageId: 'note-1' }, + { kind: 'status', text: 'a note' }, + { fence: 3, turnScope: { kind: 'thread' } } + ) + expect(journal.needsRebuild).toBe(false) + expect(stored(name)).toEqual(freshDerivation(name)) + expect(stored(name)).toMatchObject({ owesWork: true }) + }) + + it('owes the roster again once provider history rebuilds the chat', async () => { + const journal = await openCorrupt() + await journal.replaceEpochItems('legacy_import', 3, [ + { + identity: { provider: 'orca', clientMessageId: 'roster-1' }, + body: { + kind: 'message', + role: 'system', + blocks: [ + { + type: 'subagent-group', + groupId: 'group-1', + agents: [{ id: 'child-1', label: 'reads', state: 'working', startedAt: 10 }] + } + ] + } + } + ]) + expect(journal.needsRebuild).toBe(false) + expect(stored(name)).toEqual(freshDerivation(name)) + expect(stored(name)).toMatchObject({ owesWork: true }) + }) +}) + describe('a failing state write never fails the append (T2)', () => { it('commits the row, leaves the state behind the tip, and the next open re-derives it', async () => { const journal = await write('settled') const before = stored('settled') const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) - db().exec(`CREATE TEMP TRIGGER fail_state_write BEFORE UPDATE ON main.journal_session_state - BEGIN SELECT RAISE(ABORT, 'state write refused'); END`) + refuseStateWrites() await expect( journal.appendItem( @@ -139,8 +199,8 @@ describe('a failing state write never fails the append (T2)', () => { { sessionId: 'settled', current: null } ]) expect(warn).toHaveBeenCalledWith( - '[journal-append] state row skipped:', - expect.objectContaining({ sessionId: 'settled' }) + '[agent-session-journal] storing the chat state failed', + expect.objectContaining({ sessionId: 'settled', error: 'state write refused' }) ) db().exec('DROP TRIGGER fail_state_write') @@ -152,6 +212,42 @@ describe('a failing state write never fails the append (T2)', () => { }) }) +describe('a failed COMMIT leaves the fold equal to the disk (T1)', () => { + function note(journal: AgentSessionJournal, id: string) { + return journal.appendItem( + { provider: 'orca', clientMessageId: id }, + { kind: 'status', text: id }, + { fence: 3, turnScope: { kind: 'thread' } } + ) + } + + it('folds nothing, stores nothing, and the next append takes the sequence', async () => { + const journal = await write('settled') + const tip = journal.cursor() + const before = stored('settled') + const connection = db() + const exec = connection.exec.bind(connection) + let failCommit = true + vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { + if (failCommit && sql === 'COMMIT') { + failCommit = false + throw new Error('COMMIT failed: disk I/O error') + } + return exec(sql) + }) + + await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') + + expect(journal.cursor()).toEqual(tip) + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) + expect(stored('settled')).toEqual(before) + await expect(note(journal, 'kept')).resolves.toMatchObject({ + cursor: { sequence: tip.sequence + 1 } + }) + expect(stored('settled')).toEqual(freshDerivation('settled')) + }) +}) + describe('trusted only at the live tip (T9)', () => { it('leaves the schema version alone, so an older build stays writable', async () => { await write('settled') @@ -202,7 +298,7 @@ describe('trusted only at the live tip (T9)', () => { }) describe('the epoch and history transactions keep it (T10)', () => { - it('writes it for a new epoch, a roll and a replacement, inside their transactions', async () => { + it('writes it for a new epoch, a roll and a replacement, after their transactions', async () => { const journal = await open('epochs') expect(stored('epochs')).toMatchObject({ epoch: journal.epoch, @@ -227,6 +323,25 @@ describe('the epoch and history transactions keep it (T10)', () => { expect(stored('epochs')).toMatchObject({ epoch: journal.epoch, seq: 2 }) }) + it('never fails a roll or a replacement when the state write fails (R1J-2)', async () => { + const journal = await write('settled') + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + refuseStateWrites() + + await expect(journal.rollEpoch('handle_forked', 3)).resolves.toMatchObject({ sequence: 1 }) + expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(journal.epoch) + await expect( + journal.replaceEpochItems('handle_forked', 3, [ + { + identity: { provider: 'orca', clientMessageId: 'rebuilt-1' }, + body: { kind: 'status', text: 'rebuilt' } + } + ]) + ).resolves.toMatchObject({ sequence: 2 }) + expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(journal.epoch) + expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toBeNull() + }) + it('drops it with a repair, and the next open writes it again', async () => { const journal = await write('settled') const tip = journal.cursor() @@ -251,7 +366,7 @@ describe('the derivation is versioned (T11)', () => { // Pinned with the version: a change to what a row holds must bump the version, so rows an // earlier derivation wrote read as absent and are derived again. const GOLDEN = { - version: 1, + version: 2, digest: '637041b814cfbd16b3d5bd1b0e19f505bba360a3cc2d90f4ad801433b5192346' } diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index 9964acb4d132..5262ad8786f7 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -1,10 +1,11 @@ // Each chat's lifecycle state, stored beside its journal. // // One row per chat: whether its next open owes settlement work (see -// journal-open-settlement-plan.ts), and the status summary a session list shows. Written in the -// SAME transaction as the journal row it describes, and trusted only while its `(epoch, seq)` is the -// chat's live tip, so a write this build missed (an older build's append, a failed savepoint) -// reads as stale and the chat is re-derived. Deleting a row is always safe for the same reason. +// journal-open-settlement-plan.ts), and the status summary a session list shows. Written right +// after the journal commit it describes, in its own transaction, and trusted only while its +// `(epoch, seq)` is the chat's live tip, so a write that failed, was lost, or was never made (an +// older build's append) reads as stale and the chat is re-derived. Deleting a row is always safe for +// the same reason. // // Created at every writable open with no `user_version` bump, as the draft table is // (queued-message-schema.ts): an older build ignores it and stays writable. @@ -20,7 +21,7 @@ import { renderJournalState, type JournalReducerState } from './journal-reducer' /** Version of the rules that derive a row. A row of another version reads as absent. Bump it with * any change to what `deriveJournalSessionState` produces; the golden test pins it. */ -export const JOURNAL_SESSION_STATE_VERSION = 1 +export const JOURNAL_SESSION_STATE_VERSION = 2 export function ensureJournalSessionStateTable(db: Database.Database): void { db.exec(` @@ -58,6 +59,8 @@ export type JournalSessionStateInput = { /** False while the chat's load is corrupt: its open settles no roster either. */ settlesRosters: boolean currentFence?: number + /** The fold's status summary, when the caller already projected this tip. */ + statusSummary?: () => StructuredAgentSessionStatusProjection } export function deriveJournalSessionState( @@ -68,12 +71,9 @@ export function deriveJournalSessionState( let summary: StructuredAgentSessionStatusProjection | null = null if (!facts.owesWork) { // Fence-independent here: nothing unanswered or queued is left for the fence to judge. - const snapshot = renderJournalState(state) - summary = projectStructuredAgentSessionStatusState( - snapshot.items, - snapshot.submissions, - input.currentFence - ).summary + summary = input.statusSummary + ? input.statusSummary() + : projectSummary(state, input.currentFence) } return { ...facts, @@ -84,6 +84,15 @@ export function deriveJournalSessionState( } } +function projectSummary( + state: JournalReducerState, + fence: number | undefined +): StructuredAgentSessionStatusProjection { + const snapshot = renderJournalState(state) + return projectStructuredAgentSessionStatusState(snapshot.items, snapshot.submissions, fence) + .summary +} + const UPSERT_STATE = `INSERT INTO journal_session_state (session_id, state_version, epoch, seq, owes_work, unverifiable_owner_fences, summary_json, last_activity_at, written_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -179,7 +188,13 @@ WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` return results } -/** Every chat this build's rows say may owe work at open, through the partial index. */ +/** + * Every chat this build's rows say may owe work at open, through the partial index, at any position: + * a row one write behind its tip (a write that failed, or a kill between the two commits) still + * owes what it owed, since a turn running then is still running, and an open that finds nothing to + * settle appends nothing. A row that owes nothing is never selected, stale or not, so no boot opens + * a settled chat: its own open re-derives it. + */ export function readOwedJournalSessionStates( db: Database.Database ): (JournalOwedFacts & { sessionId: string })[] { @@ -202,20 +217,6 @@ WHERE (owes_work = 1 OR unverifiable_owner_fences IS NOT NULL) AND state_version ) } -/** Chats with a draft the drain would send on open: waiting, not held, and the queue unpaused. */ -export function readSessionsWithDrainableDrafts(db: Database.Database): Set { - const rows = db - .prepare( - `SELECT DISTINCT q.session_id AS session_id FROM queued_messages q -WHERE q.state = 'waiting' AND q.hold_reason IS NULL - AND NOT EXISTS (SELECT 1 FROM queued_message_pauses p WHERE p.session_id = q.session_id)` - ) - .all() - return new Set( - rows.flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) - ) -} - function parseStoredState(row: Record): StoredJournalSessionState | null { if ( row.state_version !== JOURNAL_SESSION_STATE_VERSION || @@ -284,6 +285,11 @@ function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | return typeof field === 'string' ? field : undefined } const turnOutcome = source.get('turnOutcome') + if (turnOutcome !== undefined && !isAgentJournalTurnOutcome(turnOutcome)) { + // A verdict this build cannot show: stale, so the chat's open publishes it rather than a + // row without it. + return null + } const statusStartedAt = source.get('statusStartedAt') const toolName = text('toolName') const toolInput = text('toolInput') @@ -301,8 +307,8 @@ function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | /** * The open's re-derive: rewrites the chat's row when it is absent, another version's, off the - * fold's tip, or older than a repair. What an older build, an import, a repair or a failed - * savepoint left behind is caught here. Bookkeeping: the caller logs a failure and goes on. + * fold's tip, or older than a repair. What an older build, an import, a repair or a failed or lost + * write left behind is caught here. Bookkeeping: the caller logs a failure and goes on. */ export function ensureJournalSessionStateCurrent( db: Database.Database, diff --git a/src/main/native-chat/agent-session-journal/journal-status-projection.ts b/src/main/native-chat/agent-session-journal/journal-status-projection.ts new file mode 100644 index 000000000000..9423a9bfe7ae --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-status-projection.ts @@ -0,0 +1,59 @@ +// A chat's status projection at its fold's tip, kept for one (fold, epoch, seq, fence): the status +// feed and the stored state (journal-session-state.ts) read the same one, so a commit renders and +// projects the whole chat once. + +import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import { renderJournalState, type JournalReducerState } from './journal-reducer' + +export type JournalStatusProjectionState = ReturnType< + typeof projectStructuredAgentSessionStatusState +> + +type Cached = { + fold: JournalReducerState + epoch: string + sequence: number + fence: number | undefined + state: JournalStatusProjectionState +} + +export class JournalStatusProjection { + private cached: Cached | null = null + + constructor(private readonly fold: () => JournalReducerState) {} + + /** The projection under this fence; the fence moves when a child ends, the tip does not. */ + at(fence: number | undefined): JournalStatusProjectionState { + const tip = this.atTip() + return tip && tip.fence === fence ? tip.state : this.project(fence) + } + + /** Whatever fence it was projected under: a chat that owes nothing projects the same summary + * under every fence (nothing unanswered is left for the fence to judge). */ + summaryAtTip(fence: number | undefined): JournalStatusProjectionState['summary'] { + return (this.atTip()?.state ?? this.project(fence)).summary + } + + private atTip(): Cached | null { + const fold = this.fold() + const cached = this.cached + return cached && + cached.fold === fold && + cached.epoch === fold.epoch && + cached.sequence === fold.lastSequence + ? cached + : null + } + + private project(fence: number | undefined): JournalStatusProjectionState { + const fold = this.fold() + const snapshot = renderJournalState(fold) + const state = projectStructuredAgentSessionStatusState( + snapshot.items, + snapshot.submissions, + fence + ) + this.cached = { fold, epoch: fold.epoch, sequence: fold.lastSequence, fence, state } + return state + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 6088467d614b..918bce2cd575 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -18,16 +18,13 @@ import { JournalRowWriter } from './journal-row-writer' import { restoreJournalStore } from './journal-store-restore' import type { JournalRow } from './journal-row-schema' import type { AgentSessionJournal } from './journal-store' -import type Database from '../../sqlite/sync-database' -import { applyJournalRow } from './journal-reducer' -import { readJournalRowsAfterCursor, replayJournal } from './journal-open' -import { readJournalSince } from './journal-cursor' -import type { JournalReadSince } from './journal-store-contracts' +import { JOURNAL_REPAIR_DISCLOSURE_ITEM_ID } from './journal-repair-disclosure' import { deriveJournalSessionState, ensureJournalSessionStateCurrent, writeJournalSessionState } from './journal-session-state' +import { JournalStatusProjection } from './journal-status-projection' export type JournalStoreHost = { /** Fires the journal's commit listener for a durable change that appended no @@ -50,18 +47,15 @@ export type JournalStoreHost = { setReadOnly: (readOnly: boolean) => void cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void - /** Replaces the fold alone, quietly: a failed append's re-fold from disk. */ - replaceState: (state: JournalReducerState) => void - /** The chat can no longer trust its fold or its connection: refuses every later write. */ - strand: () => void - /** Whether this open's replay found an unusable prefix. */ - openedCorrupt: () => boolean + commit: (row: JournalRow) => void + /** Whether a fresh replay would report the history corrupt. */ + loadCorrupt: () => boolean + /** Records whether the loaded history is corrupt. */ + setLoadCorrupt: (corrupt: boolean) => void /** The conversation's fence, which the stored status reads as the status feed does. */ currentFence: () => number | undefined /** A per-chat file's copy is still owed: the fold is not the database's yet. */ importPending: () => boolean - /** Records whether the open's replay found an unusable prefix. */ - setOpenedCorrupt: (corrupt: boolean) => void malformedRows: () => number setMalformedRows: (count: number) => void journal: () => AgentSessionJournal @@ -77,18 +71,15 @@ export type JournalStoreCollaborators = { /** Restores the store's state from disk. Owned here because it needs the same * collaborators the constructor just built. */ restore: () => Promise + /** Rewrites the chat's stored state if it does not describe the fold. */ ensureSessionState: () => void - readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince + statusProjection: JournalStatusProjection } export function createJournalStoreCollaborators(host: JournalStoreHost): JournalStoreCollaborators { - const writeState = (db: Database.Database, state: JournalReducerState) => - writeJournalSessionState( - db, - host.identity.sessionId, - deriveJournalSessionState(state, journalSessionStateInput(host)), - host.now() - ) + const statusProjection = new JournalStatusProjection(host.state) + const storeState = (onlyIfStale: boolean) => + storeJournalSessionState(host, statusProjection, onlyIfStale) const epochController = new JournalEpochController({ identity: host.identity, now: host.now, @@ -99,8 +90,11 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal setReadOnly: host.setReadOnly, highestFence: () => host.state().highestFence, cursor: host.cursor, - adopt: host.adopt, - writeState + adopt: (loaded) => { + host.setLoadCorrupt(loaded.corrupt) + host.adopt(loaded) + storeState(false) + } }) const queuedMessages = new JournalQueuedMessages({ sessionId: host.identity.sessionId, @@ -114,24 +108,8 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal return { epochController, queuedMessages, - ensureSessionState: () => ensureStoredSessionState(host), - readSince: (cursor, limit) => - readJournalSince( - { - state: host.state(), - rowsAfter: (afterSequence) => - readJournalRowsAfterCursor( - host.database().db, - host.identity.sessionId, - host.state().epoch, - afterSequence, - limit - ), - readOnly: host.readOnly() - }, - cursor, - host.cursor - ), + statusProjection, + ensureSessionState: () => storeState(true), // Behind the stored fact: settles drafts whose consumed submission the loaded journal shows // refused (a downgrade wrote no hook), then prunes. Bookkeeping, never failing the open. restore: () => @@ -146,10 +124,14 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal readOnly: host.readOnly, highestFence: () => host.state().highestFence, nextSequence: () => host.state().lastSequence + 1, - apply: (row) => applyJournalRow(host.state(), row), - committed: host.notifyCommitted, - recoverFold: () => recoverJournalFold(host), - writeState: (db) => writeState(db, host.state()), + commit: (row) => { + // Any row but the repair's own disclosure retires the rebuild a repair owed, as replay reads it. + if (!(row.kind === 'item' && row.itemId === JOURNAL_REPAIR_DISCLOSURE_ITEM_ID)) { + host.setLoadCorrupt(false) + } + host.commit(row) + storeState(false) + }, // Every rejection is a dispatch row through this one writer; the draft // returned-transition rides it so no path can bypass the hook. inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), @@ -167,50 +149,38 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal } } -function journalSessionStateInput(host: Pick) { - return { settlesRosters: !host.openedCorrupt(), currentFence: host.currentFence() } -} - -function ensureStoredSessionState(host: JournalStoreHost): void { +/** + * The chat's stored state for the fold a commit (or an open) left, in its own transaction after the + * journal's, unsynced: bookkeeping that never fails the write it describes. A failed or lost write + * leaves the row behind the tip, which reads as stale, and the next open or startup re-derives it. + */ +function storeJournalSessionState( + host: JournalStoreHost, + projection: JournalStatusProjection, + onlyIfStale: boolean +): void { const database = host.database() if (host.readOnly() || database.readOnly || host.importPending()) { return } + const input = { + settlesRosters: !host.loadCorrupt(), + currentFence: host.currentFence(), + statusSummary: () => projection.summaryAtTip(host.currentFence()) + } try { - ensureJournalSessionStateCurrent( - database.db, - host.state(), - journalSessionStateInput(host), - host.now() - ) + database.unsyncedTransaction((db) => { + if (onlyIfStale) { + ensureJournalSessionStateCurrent(db, host.state(), input, host.now()) + } else { + const state = deriveJournalSessionState(host.state(), input) + writeJournalSessionState(db, host.identity.sessionId, state, host.now()) + } + }) } catch (error) { - console.warn('[agent-session-journal] re-deriving the stored chat state failed', { + console.warn('[agent-session-journal] storing the chat state failed', { sessionId: host.identity.sessionId, - error + error: error instanceof Error ? error.message : String(error) }) } } - -/** - * An append whose transaction failed after its row was folded: the fold is ahead of the disk, and - * an in-place apply cannot be undone, so the chat is folded again from what committed. A stranded - * connection may still hold the uncommitted row, so a re-read would adopt it: the chat closes - * instead, and its next open reads the disk once the host database's own rollback goes through. - */ -function recoverJournalFold( - host: Pick -): void { - const database = host.database() - if (!database.isStranded) { - try { - const reloaded = replayJournal(database.db, host.identity.sessionId) - if (reloaded) { - host.replaceState(reloaded.state) - return - } - } catch (error) { - console.warn('[agent-session-journal] re-folding after a failed append failed', error) - } - } - host.strand() -} diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index 8fce8457b630..cd118c7ee78e 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -47,7 +47,7 @@ export async function restoreJournalStore( legacyDirectory: host.legacyDirectory, replay: () => { const loaded = preview ?? replayJournal(host.database().db, host.identity.sessionId) - host.setOpenedCorrupt(loaded?.corrupt ?? false) + host.setLoadCorrupt(loaded?.corrupt ?? false) return loaded }, deleteSuffix: (fromSeq, contentFrom) => @@ -99,7 +99,7 @@ async function restoreFromNewerDatabase( { cause: error } ) } - host.setOpenedCorrupt(false) + host.setLoadCorrupt(false) host.adopt({ state: loaded?.state ?? createJournalReducerState(sessionId, ''), readOnly: true, diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 77226aaed625..4523a98ad543 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -25,14 +25,16 @@ import { newestStructuredAgentSessionTurnBySequence } from '../../../shared/structured-agent-session-live-turn' import type { JournalReplacementItem } from './journal-epoch-replacement' +import { readJournalSince } from './journal-cursor' import type { JournalHostDatabase } from './journal-host-database' -import type { JournalLoad } from './journal-open' +import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open' import { markJournalPendingSubmissionsUnknown, rejectJournalPendingSubmissions, rejectJournalQueuedSubmissions } from './journal-pending-submission-recovery' import { + applyJournalRow, createJournalReducerState, renderJournalState, resolveJournalItemId, @@ -56,6 +58,10 @@ import type { ResolveDispatchInput } from './journal-store-contracts' import { queuedMessageConsumeHook, type JournalQueuedMessages } from './journal-queued-messages' +import type { + JournalStatusProjection, + JournalStatusProjectionState +} from './journal-status-projection' import type { AgentJournalEpochReason } from './journal-row-schema' import type { JournalRowWriter } from './journal-row-writer' import type { JournalEpochController } from './journal-epoch-controller' @@ -76,10 +82,10 @@ export class AgentSessionJournal { private state: JournalReducerState private readOnly = false private malformedRows = 0 - private openedCorrupt = false + /** A fresh replay would report the history corrupt: it is still owed a rebuild. */ + private loadCorrupt = false private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } private onCommitted: (() => void) | null = null - private onStranded: (() => void) | null = null private readonly queue: JournalWriteQueue private readonly rowWriter: JournalRowWriter private readonly epochController: JournalEpochController @@ -89,7 +95,7 @@ export class AgentSessionJournal { /** Rewrites the chat's stored state if it does not describe this fold. Bookkeeping: never * fails the open that calls it. */ readonly ensureSessionState: () => void - readonly readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince + private readonly statusProjection: JournalStatusProjection /** Draft rows queued while the agent works; never reducer input or owed work. */ readonly queuedMessages: JournalQueuedMessages @@ -120,18 +126,15 @@ export class AgentSessionJournal { this.adoptLoadedJournal(loaded) this.onCommitted?.() }, - replaceState: (state) => { - this.state = state - }, - strand: () => { - this.queue.markClosed() - this.onStranded?.() + commit: (row) => { + applyJournalRow(this.state, row) + this.onCommitted?.() }, - openedCorrupt: () => this.openedCorrupt, + loadCorrupt: () => this.loadCorrupt, currentFence: options.currentFence ?? (() => undefined), importPending: () => this.queue.owing, - setOpenedCorrupt: (corrupt) => { - this.openedCorrupt = corrupt + setLoadCorrupt: (corrupt) => { + this.loadCorrupt = corrupt }, notifyCommitted: () => this.onCommitted?.(), malformedRows: () => this.malformedRows, @@ -148,7 +151,7 @@ export class AgentSessionJournal { this.queuedMessages = collaborators.queuedMessages this.restore = collaborators.restore this.ensureSessionState = collaborators.ensureSessionState - this.readSince = collaborators.readSince + this.statusProjection = collaborators.statusProjection } get isReadOnly(): boolean { @@ -176,7 +179,7 @@ export class AgentSessionJournal { /** The open replayed an unusable prefix: the chat is owed a rebuild from provider history. */ get needsRebuild(): boolean { - return this.openedCorrupt + return this.loadCorrupt } async open(): Promise { @@ -197,11 +200,6 @@ export class AgentSessionJournal { this.onCommitted = listener } - /** Told once this handle closed itself because its connection was stranded mid-append. */ - observeStranded(listener: () => void): void { - this.onStranded = listener - } - /** * Resolves once the chat's rows are in the host's database. A restore's open serves a chat still * in its per-chat file from a read-only fold of it; the copy runs before the chat's first write, @@ -222,6 +220,10 @@ export class AgentSessionJournal { snapshot = (): AgentJournalSnapshot => renderJournalState(this.state) + /** The status projection at this tip, projected once per commit for every reader. */ + statusState = (fence: number | undefined): JournalStatusProjectionState => + this.statusProjection.at(fence) + /** Visits reduced items without allocating and sorting a full snapshot. */ visitItems = ( visit: (itemId: string, sequence: number, body: AgentJournalItemBody) => void @@ -287,6 +289,25 @@ export class AgentSessionJournal { canonicalItemId = (itemId: string): string => resolveJournalItemId(this.state, itemId) + readSince(cursor: AgentJournalCursor, limit?: number): JournalReadSince { + return readJournalSince( + { + state: this.state, + rowsAfter: (afterSequence) => + readJournalRowsAfterCursor( + this.database.db, + this.identity.sessionId, + this.state.epoch, + afterSequence, + limit + ), + readOnly: this.readOnly + }, + cursor, + () => this.cursor() + ) + } + /** Upsert by stable identity. The revision is assigned here so a caller * cannot accidentally publish a revision the reducer will drop. */ appendItem( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts index 085f63b32c7c..c312d6013fd5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts @@ -50,14 +50,6 @@ export class StructuredAgentSessionConversations extends Map< } }) }) - // A handle that closed itself mid-append no longer matches the disk: the next use reopens it. - // One with a child is left to that child's own sink-failure recovery. - journal.observeStranded?.(() => { - const current = this.get(sessionId) - if (current?.journal === journal && !current.child) { - this.delete(sessionId) - } - }) this.activity.set(sessionId, this.delivery.now()) const adopted = super.set(sessionId, session) this.delivery.onOpened?.(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts index 04f92fd7bdc7..f2e16ff7aaee 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -6,9 +6,15 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { + AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + type AgentJournalItemBody, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' import { createTrackedJournalOpener, + insertTestJournalRow, + insertTestJournalRowJson, liveTestJournalRows, openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' @@ -108,6 +114,88 @@ describe('the stored flag and the plan agree (T4)', () => { ).toBe(true) }) + it('agrees on a chat that opened corrupt, and again once it writes past the repair', async () => { + const first = await open('corrupt') + await JOURNAL_SESSION_STATE_CORPUS['working subagent roster'](first) + const tip = first.cursor() + await first.close() + insertTestJournalRowJson(openTestJournalHostDatabase(root).db, 'corrupt', tip.sequence + 1, '{') + const journal = await open('corrupt') + const record: OpenSettlementRecordFacts = { + sessionId: 'corrupt', + fence: CORPUS_FENCE, + deathEvidence: null + } + const plan = () => + planOpenSettlement(journal, record, { settlesRosters: !journal.needsRebuild }) + + // The rebuild is still owed, so neither touches the roster. + expect(journal.needsRebuild).toBe(true) + expect(owesOnOpen(storedFacts('corrupt'), null)).toBe(false) + expect(openSettlementPlanIsEmpty(plan())).toBe(true) + + await journal.appendItem( + { provider: 'orca', clientMessageId: 'note-1' }, + { kind: 'status', text: 'a note' }, + { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } + ) + expect(owesOnOpen(storedFacts('corrupt'), null)).toBe(true) + expect(plan().rosters).toHaveLength(1) + await appendOpenSettlement(journal, plan(), CORPUS_FENCE, (error) => { + throw error + }) + expect(owesOnOpen(storedFacts('corrupt'), null)).toBe(false) + }) + + it('owes nothing for an item whose key will not parse, which no plan can revise (R1J-4)', async () => { + const first = await open('unkeyed') + await JOURNAL_SESSION_STATE_CORPUS.settled(first) + const tip = first.cursor() + await first.close() + const unkeyed = (seq: number, itemId: string, body: AgentJournalItemBody) => + insertTestJournalRow(openTestJournalHostDatabase(root).db, 'unkeyed', { + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + kind: 'item', + itemId, + revision: 1, + body, + epoch: tip.epoch, + seq, + fence: CORPUS_FENCE, + ts: 5_000 + }) + unkeyed(tip.sequence + 1, 'legacy-tool-1', { + kind: 'tool-call', + name: 'shell', + input: { command: 'ls' }, + state: 'running' + }) + unkeyed(tip.sequence + 2, 'legacy-turn-2', { + kind: 'turn', + turnId: 't2', + state: 'running', + startedAt: 40 + }) + unkeyed(tip.sequence + 3, 'legacy-turn-3', { + kind: 'turn', + turnId: 't3', + state: 'unverifiable', + startedAt: 50 + }) + const journal = await open('unkeyed') + journal.ensureSessionState() + const deathEvidence = CORPUS_DEATH_EVIDENCE['names the writer'] ?? null + const plan = planOpenSettlement( + journal, + { sessionId: 'unkeyed', fence: CORPUS_FENCE, deathEvidence }, + { settlesRosters: true } + ) + + expect(openSettlementPlanIsEmpty(plan)).toBe(true) + expect(storedFacts('unkeyed')).toMatchObject({ owesWork: false, unverifiableOwnerFences: [] }) + expect(owesOnOpen(storedFacts('unkeyed'), deathEvidence)).toBe(false) + }) + it('owes nothing for a settled chat, whatever the record says', async () => { const journal = await open('settled') await JOURNAL_SESSION_STATE_CORPUS.settled(journal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts index d1dea753aab0..1414f85ec616 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts @@ -9,16 +9,16 @@ import { agentSessionFailureWords, type AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words' -import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { AGENT_JOURNAL_THREAD_SCOPE, type AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { partitionJournalLifecycleMutations } from '../agent-session-journal/journal-lifecycle-batch-partition' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { - isLeftoverQueuedSubmission, isRunningJournalTurn, + openSettlementItemIdentity, openSettlementTerminalBody, owesRecoveredDispatch } from '../agent-session-journal/journal-open-settlement-plan' @@ -84,7 +84,7 @@ export function planOpenSettlement( .filter(owesRecoveredDispatch) .map((submission) => submission.clientMessageId), leftoverQueued: submissions - .filter(isLeftoverQueuedSubmission) + .filter(isQueuedAgentJournalSubmission) .map((submission) => submission.clientMessageId), goneGeneration: options.acquisition ? null @@ -132,7 +132,7 @@ export function planGoneGenerationSettlement(input: { turnVerdictFromDeathEvidence(input.deathEvidence, input.itemFence(item.itemId)) const mutations: JournalLifecycleMutationInput[] = [] for (const item of items) { - const identity = parseAgentJournalItemKey(item.itemId) + const identity = openSettlementItemIdentity(item) const body = openSettlementTerminalBody(item) if (identity && body) { mutations.push({ @@ -225,8 +225,8 @@ export async function appendOpenSettlement( if (plan.recoveredDispatches.length > 0) { await journal.markPendingSubmissionsUnknown(fence) } - // Nothing indexes a conversation with a row queued (a handle closes only with none), so one - // found here was accepted by a process that is gone. + // A conversation's close abandons what it queued first, so a queued row found here was + // accepted by a process that is gone. if (plan.leftoverQueued.length > 0) { const leftovers = new Set(plan.leftoverQueued) await journal.rejectQueuedSubmissions( @@ -250,10 +250,13 @@ export async function appendOpenSettlement( } } -/** Work that means the provider was MID-RESPONSE. A pending approval or question is the provider - * waiting on the user, so dying while one sits there interrupted nothing. */ +/** Work that means the provider was MID-RESPONSE, and that this settle revises. A pending approval + * or question is the provider waiting on the user, so dying while one sits there interrupted + * nothing. */ function isInProgressItem(item: AgentJournalRenderItem): boolean { return ( - isRunningJournalTurn(item) || (item.body.kind === 'tool-call' && item.body.state === 'running') + (isRunningJournalTurn(item) || + (item.body.kind === 'tool-call' && item.body.state === 'running')) && + openSettlementItemIdentity(item) !== null ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 7eb5d8543632..7aa6f40c0033 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -196,11 +196,11 @@ describe('startup opens only what it must (T6, T7, T14)', () => { } await crashMidSend(rig, 'session-crashed') await crashMidSend(rig, 'session-crashed-closed', false) + await crashMidSend(rig, 'session-crashed-state-lost', false) for (const sessionId of [ 'session-stale', 'session-repaired', 'session-draft', - 'session-paused', 'session-uncopied' ]) { await restTestChat(rig, sessionId, { message: sessionId }) @@ -213,18 +213,22 @@ describe('startup opens only what it must (T6, T7, T14)', () => { ?.epoch ) const tip = (sessionId: string) => readJournalSessionState(db(rig), sessionId)! + const appendPastState = (sessionId: string) => + insertTestJournalRow(db(rig), sessionId, { + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + kind: 'item', + itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'unstored-note' }), + revision: 1, + body: { kind: 'status', text: 'no state stored beside it' }, + epoch: live(sessionId), + seq: tip(sessionId).seq + 1, + fence: 1, + ts: 9_000 + }) // An older build appended past the stored state. - insertTestJournalRow(db(rig), 'session-stale', { - v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, - kind: 'item', - itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'older-build-note' }), - revision: 1, - body: { kind: 'status', text: 'from an older build' }, - epoch: live('session-stale'), - seq: tip('session-stale').seq + 1, - fence: 1, - ts: 9_000 - }) + appendPastState('session-stale') + // A kill between a commit and its state write: the row is one behind, still owing. + appendPastState('session-crashed-state-lost') // An older build repaired the chat after its state was stored, and wrote back to that sequence. db(rig) .prepare( @@ -236,23 +240,16 @@ describe('startup opens only what it must (T6, T7, T14)', () => { tip('session-repaired').seq, tip('session-repaired').writtenAt + 1 ) - const draft = db(rig).prepare( - `INSERT INTO queued_messages (session_id, message_id, position, body_json, fingerprint, - created_at, host_instance, state) VALUES (?, 'draft-1', 1, ?, 'fp', 1, 'host-a', 'waiting')` - ) - const body = JSON.stringify({ - kind: 'message', - role: 'user', - blocks: [{ type: 'text', text: 'x' }] - }) - draft.run('session-draft', body) - draft.run('session-paused', body) + // A draft an earlier process queued: paused by the restart, so an open would send nothing. db(rig) .prepare( - `INSERT INTO queued_message_pauses (session_id, reason, epoch, sequence, recorded_at) - VALUES ('session-paused', 'stopped', ?, 1, 1)` + `INSERT INTO queued_messages (session_id, message_id, position, body_json, fingerprint, + created_at, host_instance, state) VALUES (?, 'draft-1', 1, ?, 'fp', 1, 'host-a', 'waiting')` + ) + .run( + 'session-draft', + JSON.stringify({ kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'x' }] }) ) - .run(live('session-paused')) // Still in the format a pre-database build wrote: nothing of it is in the host's database. for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { db(rig).prepare(`DELETE FROM ${table} WHERE session_id = ?`).run('session-uncopied') @@ -269,21 +266,22 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(listed).not.toContain('session-crashed-closed') const background = await startup(rig, listed) - expect(opened(rig, [...listed, 'session-crashed-closed']).toSorted()).toEqual( + const unlisted = ['session-crashed-closed', 'session-crashed-state-lost'] + expect(opened(rig, [...listed, ...unlisted]).toSorted()).toEqual( [ 'session-crashed', - 'session-crashed-closed', - 'session-draft', + ...unlisted, 'session-repaired', 'session-stale', 'session-uncopied' ].toSorted() ) expect(background.toSorted()).toEqual( - ['session-draft', 'session-repaired', 'session-stale', 'session-uncopied'].toSorted() + ['session-repaired', 'session-stale', 'session-uncopied'].toSorted() ) + expect(latestRestTestStatus(rig, 'session-draft')).toMatchObject({ status: 'idle' }) // Settled with no user action; the listed one is open and never showed its pre-crash work. - for (const sessionId of ['session-crashed', 'session-crashed-closed']) { + for (const sessionId of ['session-crashed', ...unlisted]) { // The unanswered send is now recovered doubt, which projects as no running request. expect(readJournalSessionState(db(rig), sessionId)).toMatchObject({ owesWork: false, @@ -292,11 +290,11 @@ describe('startup opens only what it must (T6, T7, T14)', () => { } expect(rig.host.hasSession('session-crashed')).toBe(true) expect(statusRows(rig, 'session-crashed').map((row) => row.status)).not.toContain('working') - // The tabless one is settled and closed: never indexed, never given a status row (T14). - expect(rig.host.hasSession('session-crashed-closed')).toBe(false) - expect( - rig.sink.publish.mock.calls.filter(([row]) => row.sessionId === 'session-crashed-closed') - ).toEqual([]) + // The tabless ones are settled and closed: never indexed, never given a status row (T14). + for (const sessionId of unlisted) { + expect(rig.host.hasSession(sessionId)).toBe(false) + expect(rig.sink.publish.mock.calls.filter(([row]) => row.sessionId === sessionId)).toEqual([]) + } // Every settled chat has its row without being opened. for (const sessionId of settled) { expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) @@ -306,10 +304,7 @@ describe('startup opens only what it must (T6, T7, T14)', () => { await rig.crash() await rig.boot() const again = await startup(rig, listedIds(rig)) - expect(opened(rig, [...listedIds(rig), 'session-crashed-closed']).toSorted()).toEqual( - again.toSorted() - ) - // Only the chat whose draft still waits is opened again, as every listed chat used to be. - expect(again).toEqual(['session-draft']) + expect(again).toEqual([]) + expect(opened(rig, [...listedIds(rig), ...unlisted])).toEqual([]) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index cbfb93caca57..58668e49843c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -6,8 +6,8 @@ // not, is opened once, one at a time, and its open appends the settlement plan. A listed one goes // through the restart restore's own per-chat worker and stays open; an unlisted one is settled and // closed, never indexed or published. Everything else a listed chat needs (a stale row, a per-chat -// file not yet copied, a draft the drain would send) is left to the background restore after the -// listing, which is the same worker. +// file not yet copied) is left to the background restore after the listing, which is the same +// worker. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' @@ -16,8 +16,7 @@ import type { AgentSessionRecordStore } from '../../runtime/agent-session-record import { owesOnOpen } from '../agent-session-journal/journal-open-settlement-plan' import { readJournalSessionStatesAtTip, - readOwedJournalSessionStates, - readSessionsWithDrainableDrafts + readOwedJournalSessionStates } from '../agent-session-journal/journal-session-state' import { openStructuredAgentSessionConversationJournal, @@ -71,10 +70,8 @@ function seedStoredStatuses( return [...listedIds] } let states: ReturnType - let drafts: Set try { states = readJournalSessionStatesAtTip(database.db, listedIds) - drafts = readSessionsWithDrainableDrafts(database.db) } catch (error) { // Fails open: every listed chat is restored in the background, as before stored state existed. console.warn('[structured-agent-session] reading stored chat state failed', error) @@ -111,10 +108,6 @@ function seedStoredStatuses( continue } deps.seedStatus(record, { projected: stored.summary, lastActivityAt: stored.lastActivityAt }) - if (drafts.has(sessionId)) { - // Opened so the drain runs on open, as the restore always did. - background.push(sessionId) - } } return background } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index ad23177bdd3a..b91718265bc2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -636,25 +636,25 @@ describe('StructuredAgentSessionStatusFeed', () => { { kind: 'status', text: 'Working', turnLifecycle: { turnId: 'turn-1', state: 'running' } }, { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) - const snapshot = vi.spyOn(journal, 'snapshot') let taskState: 'working' | 'waiting' = 'working' const { feed, events } = feedFor(new Map([[SESSION, { journal }]]), null, undefined, () => ({ state: 'monitoring', tasks: [{ id: 'child', kind: 'agent', state: taskState }] })) + const projected = feed.statusState(SESSION) for (let tick = 1; tick <= 100; tick++) { taskState = tick % 2 === 1 ? 'waiting' : 'working' feed.publish(SESSION) + expect(feed.statusState(SESSION)).toBe(projected) } expect(events).toHaveLength(101) - expect(snapshot).toHaveBeenCalledTimes(1) expect(events.at(-1)).toMatchObject({ type: 'status', session: { status: 'working', backgroundTasks: [{ state: 'working' }] } }) await journal.appendTombstone(TURN_IDENTITY, { fence: 1 }) feed.publish(SESSION) - expect(snapshot).toHaveBeenCalledTimes(2) + expect(feed.statusState(SESSION)).not.toBe(projected) expect(events.at(-1)).toMatchObject({ type: 'status', session: { status: 'idle' } }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index efc1def8958e..91264592aec2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -44,8 +44,11 @@ export type StructuredAgentSessionStatusSubscriber = { emit: (event: AgentSessionStatusEvent) => void } +/** All the feed reads of a chat's journal. */ +type StatusFeedJournal = Pick + type StatusFeedSession = { - journal: AgentSessionJournal + journal: StatusFeedJournal params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } child?: Pick | null } @@ -104,17 +107,6 @@ export class StructuredAgentSessionStatusFeed { ) private readonly subscribers = new Map() private readonly published = new Map() - // Task progress must not sort and scan an unchanged conversation. Journal identity owns cleanup. - private readonly journalProjections = new WeakMap< - AgentSessionJournal, - { - epoch: string - sequence: number - readOnly: boolean - fence: number | undefined - state: StructuredAgentSessionStatusState - } - >() constructor(private readonly deps: StructuredAgentSessionStatusFeedDeps) {} @@ -183,7 +175,7 @@ export class StructuredAgentSessionStatusFeed { * so the completion feed follows the same request without snapshotting the journal again. */ statusState( sessionId: string, - journal?: AgentSessionJournal + journal?: StatusFeedJournal ): StructuredAgentSessionStatusState | null { const session = this.deps.sessions.get(sessionId) const source = journal ?? session?.journal @@ -191,7 +183,7 @@ export class StructuredAgentSessionStatusFeed { } /** Re-projects one session after its journal changed; equal projections are not re-sent. */ - publish(sessionId: string, journal?: AgentSessionJournal, options?: { replay?: boolean }): void { + publish(sessionId: string, journal?: StatusFeedJournal, options?: { replay?: boolean }): void { const session = this.deps.sessions.get(sessionId) if (!session) { return @@ -233,7 +225,7 @@ export class StructuredAgentSessionStatusFeed { private summaryFor( sessionId: string, session: StatusFeedSession, - journal: AgentSessionJournal + journal: StatusFeedJournal ): AgentSessionStatusSummary { const record = this.deps.getRecord(sessionId) return structuredAgentSessionStatusSummary({ @@ -291,38 +283,16 @@ export class StructuredAgentSessionStatusFeed { } private projectionFor( - journal: AgentSessionJournal, + journal: StatusFeedJournal, record: AgentSessionRecord | null ): StructuredAgentSessionStatusState { - // An unreadable journal projects as "no turn": the chat itself shows the reset. - const cursor = journal.cursor() - const readOnly = journal.isReadOnly // The conversation's fence, which a child's end moves: its unanswered sends stop counting. const fence = record?.lease.runtimeFence - let projection = this.journalProjections.get(journal) - if ( - !projection || - projection.epoch !== cursor.epoch || - projection.sequence !== cursor.sequence || - projection.readOnly !== readOnly || - projection.fence !== fence - ) { - // A journalled submission bumps `lastSequence`, so the send-time working - // signal reaches the cache; the lease fence does not, hence the extra key. - const snapshot = readOnly ? null : journal.snapshot() - projection = { - ...cursor, - readOnly, - fence, - state: projectStructuredAgentSessionStatusState( - snapshot?.items ?? [], - snapshot?.submissions ?? [], - fence - ) - } - this.journalProjections.set(journal, projection) - } - return projection.state + // An unreadable journal projects as "no turn": the chat itself shows the reset. Otherwise + // the journal's own projection, once per commit and fence, which its stored state reads too. + return journal.isReadOnly + ? projectStructuredAgentSessionStatusState([], [], fence) + : journal.statusState(fence) } /** A failing sink must never cost the subscribers their status event. */ diff --git a/src/main/orcad/orcad-push-startup.test.ts b/src/main/orcad/orcad-push-startup.test.ts index 21ad507c0164..390346fe3018 100644 --- a/src/main/orcad/orcad-push-startup.test.ts +++ b/src/main/orcad/orcad-push-startup.test.ts @@ -74,6 +74,7 @@ vi.mock('../runtime/orca-runtime', () => ({ rehydrateClientHostedBrowserPages() {} async refreshRestoredOrchestrationAuthority() {} async reconcileLegacyWorkerTerminals() {} + async prepareStructuredAgentSessionStartupRestoration() {} setMobilePushRegistrar( registrar: Parameters[0] ) { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index 85d3e2a5b4bf..f75769363829 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -4,7 +4,7 @@ import { vi } from 'vitest' import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' -import type { AgentSessionJournal } from '../../../native-chat/agent-session-journal/journal-store' +import { projectStructuredAgentSessionStatusState } from '../../../../shared/structured-agent-session-projection' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import { @@ -100,10 +100,10 @@ function statusFeed(): StructuredAgentSessionStatusFeed { { journal: { isReadOnly: false, - cursor: () => ({ epoch: 'epoch-status', sequence: 2 }), lastActivityAt: () => 2, - snapshot: () => ({ items: STATUS_ITEMS }) - } as unknown as AgentSessionJournal, + statusState: (fence?: number) => + projectStructuredAgentSessionStatusState(STATUS_ITEMS, [], fence) + }, params: { location: { executionHostId: 'local', From 4a05b815f0b110ec17b8746cebe35d3de91a167c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:54:33 -0700 Subject: [PATCH 024/107] fix(native-chat): seeded rows keep a crash-cut turn's verdict; type the startup step A chat's stored summary now reads the turn verdict with the shared predicate, so a turn a crash cut off seeds as Interrupted or Couldn't confirm, as its open publishes it, and a verdict this build cannot read makes the row stale rather than dropping the field. The derivation version is bumped and re-pinned. The startup step (lease check, seed, settle, listed chats) moves out of the untyped runtime file into a typed module named from the host's own members. --- .../journal-session-state.test.ts | 2 +- .../journal-session-state.ts | 6 +- ...ctured-agent-session-startup-state.test.ts | 56 +++++++++++++++ ...structured-agent-session-create-support.ts | 41 ++--------- ...tore-structured-agent-session-tabs-once.ts | 11 ++- .../structured-agent-session-startup-step.ts | 69 +++++++++++++++++++ 6 files changed, 143 insertions(+), 42 deletions(-) create mode 100644 src/main/runtime/structured-agent-session-startup-step.ts diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 30dab49af63e..07a3b9149f1d 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -367,7 +367,7 @@ describe('the derivation is versioned (T11)', () => { // earlier derivation wrote read as absent and are derived again. const GOLDEN = { version: 2, - digest: '637041b814cfbd16b3d5bd1b0e19f505bba360a3cc2d90f4ad801433b5192346' + digest: '33a151ec037c24cbd1bd74cfaacc0bea419d132e1c4878724e416294635bba9b' } it('matches the digest pinned beside its version', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index 5262ad8786f7..79a19130b7dd 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -11,7 +11,7 @@ // (queued-message-schema.ts): an older build ignores it and stays writable. import type Database from '../../sqlite/sync-database' -import { isAgentJournalTurnOutcome } from '../../../shared/agent-turn-outcome' +import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' import { projectStructuredAgentSessionStatusState, type StructuredAgentSessionStatusProjection @@ -285,7 +285,7 @@ function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | return typeof field === 'string' ? field : undefined } const turnOutcome = source.get('turnOutcome') - if (turnOutcome !== undefined && !isAgentJournalTurnOutcome(turnOutcome)) { + if (turnOutcome !== undefined && !isAgentTurnOutcome(turnOutcome)) { // A verdict this build cannot show: stale, so the chat's open publishes it rather than a // row without it. return null @@ -300,7 +300,7 @@ function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | ...(toolName !== undefined ? { toolName } : {}), ...(toolInput !== undefined ? { toolInput } : {}), ...(lastAssistantMessage !== undefined ? { lastAssistantMessage } : {}), - ...(isAgentJournalTurnOutcome(turnOutcome) ? { turnOutcome } : {}), + ...(isAgentTurnOutcome(turnOutcome) ? { turnOutcome } : {}), ...(typeof statusStartedAt === 'number' ? { statusStartedAt } : {}) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 7aa6f40c0033..17f3c12f0ffb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -114,6 +114,62 @@ describe('seeding statuses from stored state', () => { expect(ids.map((sessionId) => statusRows(rig, sessionId).length)).toEqual(seeded) }) + it('seeds a crash-cut turn with the verdict its open publishes (T5, interrupted and unconfirmed)', async () => { + const rig = await newRig() + const ids = ['session-interrupted', 'session-unconfirmed'] + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + const [{ providerIdentity }] = await Promise.all( + rig.adapter.dispatch.mock.results.slice(-1).map((result) => result.value) + ) + const session = rig.host.collaboratorsForTests().sessions.get(sessionId)! + // The turn's own row, beside the accepted message the dispatch's identity names. + await session.journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { + fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, + turnScope: { kind: 'thread' } + } + ) + } + await rig.crash() + // The unconfirmed chat's owner can never be judged; the other's is proven gone. + const probe = async (record: { sessionId: string }) => + record.sessionId === 'session-unconfirmed' + ? { outcome: 'indeterminate' as const, reason: 'no start time' } + : { outcome: 'pid-absent' as const } + rig.probeOwner.mockImplementation(probe) + await rig.boot() + await startup(rig, ids) + await rig.host.flushAllStreamedEvents() + await rig.crash() + closeTestJournalHostDatabases() + const readRoot = `${rig.root}-read` + await cp(rig.root, readRoot, { recursive: true }) + const readRig = await newRig(readRoot) + readRig.probeOwner.mockImplementation(probe) + + const host = await rig.boot() + await host.reconcileRestartLeases() + expect(host.seedStoredStatuses(ids)).toEqual([]) + expect(opened(rig, ids)).toEqual([]) + await readRig.host.reconcileRestartLeases() + for (const sessionId of ids) { + await readRig.host.history({ sessionId, direction: 'tail' }) + } + + for (const sessionId of ids) { + expect(latestRestTestStatus(rig, sessionId)).toEqual(latestRestTestStatus(readRig, sessionId)) + } + expect(latestRestTestStatus(rig, 'session-interrupted')).toMatchObject({ + turnOutcome: 'interruption' + }) + expect(latestRestTestStatus(rig, 'session-unconfirmed')).toMatchObject({ + turnOutcome: 'unconfirmed' + }) + }) + it('has every settled chat in the first snapshot a later subscriber gets (T8, T17)', async () => { const rig = await newRig() for (const sessionId of ['session-1', 'session-2']) { diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index 8b5512ad8aea..2cb0b563f88e 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -10,7 +10,7 @@ import { resolveStructuredAgentSessionAdoptionForCreate } from './structured-agent-session-create-adoption' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' -import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' +import { runStructuredAgentSessionStartupStep } from './structured-agent-session-startup-step' import { getLocalProjectWorktreeGitOptions } from '../project-runtime-git-options' import type { AgentSessionAttachParams } from '../native-chat/agent-session-wire/structured-agent-session-attach' import { resolveTuiAgentLaunchEnv } from '../../shared/tui-agent-launch-defaults' @@ -288,41 +288,12 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR ) await this.refreshMobileSessionPtyRecords() const host = getStructuredAgentSessionHost() - // Death evidence the lease check writes must exist before the settle takes its verdicts. A - // failed check leaves them `unverifiable`, which is safe, so the steps below still run. - let leaseFailure: { error: unknown } | null = null - try { - await host?.reconcileRestartLeases() - } catch (error) { - leaseFailure = { error } - } - if (typeof host?.seedStoredStatuses === 'function') { - const listedIds = this.listedStructuredAgentSessionIds(host) - this.structuredAgentSessionBackgroundRestoreIds = host.seedStoredStatuses(listedIds) - // Un-awaited: nothing waits on a crashed chat's settle, and it never rejects. - void host.settleOwedSessions(listedIds) - } - if (leaseFailure) { - throw leaseFailure.error - } - } - - /** The chats with a tab, for the startup step and the tab restore alike: the host's persisted - * tab index, or before it is recorded, the saved workspace session's. */ - protected listedStructuredAgentSessionIds(host): string[] { - const persistedVisibleIndex = - typeof host?.getPersistedVisibleSessionTabIndex === 'function' - ? host.getPersistedVisibleSessionTabIndex() - : { present: false, sessionIds: [] } - if (persistedVisibleIndex.present) { - return persistedVisibleIndex.sessionIds + if (host) { + this.structuredAgentSessionBackgroundRestoreIds = await runStructuredAgentSessionStartupStep( + host, + this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + ) } - const profileIds = collectSavedStructuredAgentSessionIds( - this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null - ) - // Unrecorded, the profile's chats join the tabs chats opened while the import was owed left. - // First: after a /clear the profile's chat would take their tab id, so seeds hit tabIdTaken. - return [...new Set([...persistedVisibleIndex.sessionIds, ...profileIds])] } protected hasPersistedStructuredAgentSessionStore(): boolean { diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index db5c9f086f8a..4e14fe7a0dae 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -5,6 +5,8 @@ import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import { seedStructuredAgentSessionTabIndex } from './structured-agent-session-tab-index-seed' +import { listedStructuredAgentSessionIds } from './structured-agent-session-startup-step' +import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import type { RuntimeMobileSessionAgentTab, RuntimeMobileSessionTabsSnapshot, @@ -51,7 +53,10 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu protected async restoreStructuredAgentSessionTabsOnce(): Promise { await this.prepareStructuredAgentSessionStartupRestoration() const host = getStructuredAgentSessionHost() - const listedIds = this.listedStructuredAgentSessionIds(host) + const listedIds = listedStructuredAgentSessionIds( + host, + this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + ) for (const worktreeId of this.getKnownWorkspaceSessionWorktreeIds()) { this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { allowAttachedWindow: true, @@ -85,8 +90,8 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } // Startup already seeded every settled chat's row and is settling the ones that owe work; this - // opens only what stored state could not answer (a stale row, a per-chat file not yet copied, a - // draft to drain). Whoever answers with this list starts it, once that answer is out. + // opens only what stored state could not answer (a stale row, a per-chat file not yet copied). + // Whoever answers with this list starts it, once that answer is out. const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds this.owedStructuredAgentSessionHistoryRestore = host ? () => diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts new file mode 100644 index 000000000000..0a19da5a0ab7 --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -0,0 +1,69 @@ +// The structured-chat step of host startup, before any client lists a tab: the restart lease check, +// then each listed chat's status seeded from its stored state, and every chat that owes work +// settled. + +import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' + +/** Named from the host so a rename fails to compile. Only the lease check is required: a host + * stand-in without stored state seeds and settles nothing, and lists the saved session's chats. */ +export type StructuredAgentSessionStartupHost = Pick< + StructuredAgentSessionHost, + 'reconcileRestartLeases' +> & + StructuredAgentSessionListingHost & + Partial> + +type StructuredAgentSessionListingHost = Partial< + Pick +> + +/** + * Answers the listed chats stored state could not answer, which the post-listing restore opens. + * Death evidence the lease check writes must exist before the settle takes its verdicts; a failed + * check leaves them `unverifiable`, which is safe, so the seed and settle still run and the failure + * is thrown after them. + */ +export async function runStructuredAgentSessionStartupStep( + host: StructuredAgentSessionStartupHost, + savedSession: WorkspaceSessionState | null +): Promise { + let leaseFailure: { error: unknown } | null = null + try { + await host.reconcileRestartLeases() + } catch (error) { + leaseFailure = { error } + } + const listedIds = listedStructuredAgentSessionIds(host, savedSession) + const background = host.seedStoredStatuses?.(listedIds) ?? listedIds + // Un-awaited: nothing waits on a crashed chat's settle, and it never rejects. + void host.settleOwedSessions?.(listedIds) + if (leaseFailure) { + throw leaseFailure.error + } + return background +} + +/** The chats with a tab, for the startup step and the tab restore alike: the host's persisted tab + * index, or before it is recorded, the saved workspace session's. */ +export function listedStructuredAgentSessionIds( + host: StructuredAgentSessionListingHost | null, + savedSession: WorkspaceSessionState | null +): string[] { + const persistedVisibleIndex = host?.getPersistedVisibleSessionTabIndex?.() ?? { + present: false, + sessionIds: [] + } + if (persistedVisibleIndex.present) { + return persistedVisibleIndex.sessionIds + } + // Unrecorded, the profile's chats join the tabs chats opened while the import was owed left. + // First: after a /clear the profile's chat would take their tab id, so seeds hit tabIdTaken. + return [ + ...new Set([ + ...persistedVisibleIndex.sessionIds, + ...collectSavedStructuredAgentSessionIds(savedSession) + ]) + ] +} From 4d4a7b074da61ba1c114046606d192ac70089bde Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:12:14 -0700 Subject: [PATCH 025/107] perf(native-chat): judge each chat item once for the owed flag, not on every append A write replaces the item object it revises and an item's creating fence never changes, so what an item owes is kept per object and each append judges only the items it touched. --- .../journal-open-settlement-plan.ts | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts index af363fbc3a3d..1826f18fab9d 100644 --- a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts +++ b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts @@ -87,22 +87,43 @@ export function owesOpenSettlement( const fences = new Set() const itemFence = (itemId: string) => fold.itemFences.get(itemId) for (const item of fold.items.values()) { - owesWork ||= options.settlesRosters && staleSubagentRosterRevision(item) !== null - const fence = unverifiableTurnOwnerFence(item, itemFence) - const settles = - fence !== undefined || isRunningJournalTurn(item) || openSettlementTerminalBody(item) !== null - if (!settles || !openSettlementItemIdentity(item)) { - continue - } - if (fence === undefined) { - owesWork = true - } else { - fences.add(fence) + const owed = itemOwes(item, itemFence) + owesWork ||= owed.settles || (options.settlesRosters && owed.roster) + if (owed.unverifiableOwnerFence !== undefined) { + fences.add(owed.unverifiableOwnerFence) } } return { owesWork, unverifiableOwnerFences: [...fences].sort((a, b) => a - b) } } +type ItemOwes = { settles: boolean; roster: boolean; unverifiableOwnerFence: number | undefined } + +// Per item object: a write replaces the object it revises, and an item's creating fence never +// changes, so each append judges only the items it touched. +const ITEM_OWES = new WeakMap() + +function itemOwes( + item: AgentJournalRenderItem, + itemFence: (itemId: string) => number | undefined +): ItemOwes { + let owes = ITEM_OWES.get(item) + if (!owes) { + const fence = unverifiableTurnOwnerFence(item, itemFence) + const revisable = + (fence !== undefined || + isRunningJournalTurn(item) || + openSettlementTerminalBody(item) !== null) && + openSettlementItemIdentity(item) !== null + owes = { + settles: revisable && fence === undefined, + roster: staleSubagentRosterRevision(item) !== null, + unverifiableOwnerFence: revisable ? fence : undefined + } + ITEM_OWES.set(item, owes) + } + return owes +} + /** The record-dependent rule, as the plan applies it: an `unverifiable` turn is revised only by * death evidence naming its writer. An older build's evidence names no writer, so selects none. */ export function owesOnOpen( From 058b5519c4d13381facbfa175c67b55e9fc77a9a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:06:19 -0700 Subject: [PATCH 026/107] refactor(native-chat): each chat's status row, written with its journal, settled behind a startup gate The chat's status row is written in the same transaction as every journal write, from the fold that write produces, with no savepoint: a failed status write fails the write. The append folds inside its transaction and, if the transaction fails, re-reads the chat from disk (a failed re-read marks the fold stale and it is re-read before its next use). Submissions are replaced, never changed in place, so a held one never shows a rolled-back change. Epoch start, roll and replacement, the per-chat file copy and the corruption repair write the row in their own transactions. The row holds facts, not an "owes work" flag: running work, a waiting prompt, handed-over and queued sends, live child work, the active turn, the summary. It arrives with schema version 5, so the position check, the repair guard and the rules version go away. Startup first resolves every lease a crash left recovering (a surviving provider process is stopped), then settles every chat whose row shows such work, in one step that every chat command waits for; listing, paint and status reads do not wait, and the gate always opens. --- docs/reference/agent-status-store.md | 6 +- ...journal-database-records-migration.test.ts | 41 ++- .../journal-database-schema.ts | 5 +- .../agent-session-journal/journal-database.ts | 30 +- .../journal-dispatch-reducer.ts | 7 +- .../journal-epoch-controller.ts | 6 +- .../journal-epoch-replacement.test.ts | 1 + .../journal-epoch-replacement.ts | 8 +- .../journal-epoch-rollover.ts | 41 ++- .../journal-host-database-test-support.ts | 12 + .../journal-open-settlement-plan.ts | 115 ++---- .../journal-per-session-import.test.ts | 26 +- .../journal-per-session-import.ts | 5 +- .../journal-repair-marker.ts | 6 +- .../journal-row-writer.test.ts | 7 +- .../journal-row-writer.ts | 25 +- .../journal-session-state-test-corpus.ts | 6 +- .../journal-session-state.test.ts | 346 +++++++----------- .../journal-session-state.ts | 329 +++++++---------- .../journal-status-projection.ts | 6 - .../journal-store-collaborators.ts | 128 +++++-- .../journal-store-restore.ts | 4 +- .../agent-session-journal/journal-store.ts | 62 ++-- .../journal-submission-fold.ts | 7 +- ...red-agent-session-conversation-lifetime.ts | 5 + ...uctured-agent-session-conversation-open.ts | 2 +- .../structured-agent-session-host-delivery.ts | 11 +- .../structured-agent-session-host-types.ts | 3 + ...ured-agent-session-open-settlement.test.ts | 76 ++-- .../structured-agent-session-reveal.ts | 1 + ...ctured-agent-session-startup-state.test.ts | 201 +++++++--- .../structured-agent-session-startup-state.ts | 100 ++--- src/main/orcad/orcad-entry.ts | 4 +- src/main/orcad/orcad-push-startup.test.ts | 1 + ...structured-agent-session-create-support.ts | 41 ++- .../runtime/orca-runtime-get-worktree-ps.ts | 12 +- ...gent-session-record-import-install.test.ts | 5 +- .../structured-agent-session-runtime.ts | 3 + .../structured-agent-session-startup-gate.ts | 54 +++ ...-agent-session-startup-restoration.test.ts | 48 ++- .../structured-agent-session-startup-step.ts | 12 +- ...-process-runtime-launch-activation.test.ts | 6 +- .../startup/main-process-runtime-launch.ts | 6 +- 43 files changed, 1009 insertions(+), 811 deletions(-) create mode 100644 src/main/runtime/structured-agent-session-startup-gate.ts diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 54585a5f6bd4..b0f7dc65415e 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -131,9 +131,9 @@ Two rules the ingest must keep: - **Never persist a structured row in the store.** The chat journal database is the durable truth for a structured session. Each chat's status is stored there beside its journal (`journal_session_state`), written in the same - transaction as the journal rows it describes and trusted only at the - journal's tip. At host startup the host republishes those rows into the store - without opening the chat. A structured row in `last-status.json` would + transaction as the journal rows it describes, so it is always current. At + host startup the host republishes a settled chat's row into the store without + opening the chat, and settles a chat a gone process left with work first. A structured row in `last-status.json` would hydrate as `restoredUnconfirmed` and fight that republish, so the serializer still skips rows carrying `structuredHost`, and hydrate still drops any found on disk. Applying one therefore also skips diff --git a/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts b/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts index 120588b306dd..e8363cec4ced 100644 --- a/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts @@ -8,13 +8,14 @@ import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import Database from '../../sqlite/sync-database' import { + journalDatabaseMigratesRecords, journalPragmaNumber, NO_LEGACY_JOURNAL_RECORDS, openJournalDatabase, readJournalDatabaseVersion, type JournalLegacyRecordImport } from './journal-database' -import { createJournalTablesSql } from './journal-database-schema' +import { createJournalTablesSql, JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' import { journalDatabasePath } from './journal-host-database' let root: string @@ -78,7 +79,7 @@ const hasTable = (db: Database.Database, name: string): boolean => .get(name) !== undefined describe('the version-4 migration', () => { - it('moves a version-3 database to 4 with its rows and the copied records', () => { + it('moves a version-3 database to the current version with its rows and the copied records', () => { seedVersion3() const copy = importOf('session-a') @@ -88,7 +89,7 @@ describe('the version-4 migration', () => { expect(opened).toMatchObject({ readOnly: false, legacyRecordImportOwed: false }) expect(copy.runs).toBe(1) inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(4) + expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(recordIds(db)).toEqual(['session-a']) expect(db.prepare('SELECT epoch FROM journal_sessions').get()).toEqual({ epoch: 'e1' }) }) @@ -113,7 +114,7 @@ describe('the version-4 migration', () => { sql: string, options?: { simple?: boolean } ) { - if (sql === 'user_version = 4') { + if (sql === `user_version = ${JOURNAL_DB_SCHEMA_VERSION}`) { throw new Error('crash before the version is published') } return original.call(this, sql, options) @@ -132,7 +133,7 @@ describe('the version-4 migration', () => { openJournalDatabase(dbPath, retry).db.close() expect(retry.runs).toBe(1) inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(4) + expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(recordIds(db)).toEqual(['session-a']) }) }) @@ -151,7 +152,7 @@ describe('the version-4 migration', () => { inspect((db) => expect(journalPragmaNumber(db, 'user_version')).toBe(3)) openJournalDatabase(dbPath, importOf('session-a')).db.close() inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(4) + expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(recordIds(db)).toEqual(['new', 'session-a']) }) }) @@ -172,11 +173,11 @@ describe('the version-4 migration', () => { expect(existsSync(dbPath)).toBe(false) }) - it('creates every table on a fresh file at version 4', () => { + it('creates every table on a fresh file at the current version', () => { openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db.close() inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(4) + expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) for (const table of [ 'journal_rows', 'agent_session_records', @@ -206,8 +207,30 @@ function openAsVersion3Build(path: string): { db: Database.Database; readOnly: b return { db: probe, readOnly: false } } +describe('version 5, each chat status', () => { + it('moves a version-4 database to 5 without copying the records again, with an empty status table', () => { + seedVersion3() + openJournalDatabase(dbPath, importOf('session-a')).db.close() + inspect((db) => { + db.pragma('user_version = 4') + db.exec('DROP TABLE journal_session_state') + }) + expect(journalDatabaseMigratesRecords(4)).toBe(false) + const again = importOf('session-b') + + openJournalDatabase(dbPath, again).db.close() + + expect(again.runs).toBe(0) + inspect((db) => { + expect(journalPragmaNumber(db, 'user_version')).toBe(5) + expect(hasTable(db, 'journal_session_state')).toBe(true) + expect(recordIds(db)).toEqual(['session-a']) + }) + }) +}) + describe('a build from before the move', () => { - it('opens a version-4 database read-only, so it never writes beside the records', () => { + it('opens a current database read-only, so it never writes beside the records', () => { openJournalDatabase(dbPath, importOf('session-a')).db.close() const older = openAsVersion3Build(dbPath) diff --git a/src/main/native-chat/agent-session-journal/journal-database-schema.ts b/src/main/native-chat/agent-session-journal/journal-database-schema.ts index e110b9221c44..6704fb817f1c 100644 --- a/src/main/native-chat/agent-session-journal/journal-database-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-database-schema.ts @@ -12,7 +12,10 @@ /** DB shape version, carried in `PRAGMA user_version`. Independent of the row body version * (`JournalRow.v`): a newer build can change either alone. A newer version opens read-only here, * so every change stays additive. */ -export const JOURNAL_DB_SCHEMA_VERSION = 4 +export const JOURNAL_DB_SCHEMA_VERSION = 5 + +/** The version whose migration copies the chat records file in: "copied" is `user_version >= 4`. */ +export const JOURNAL_DB_RECORDS_VERSION = 4 /** The first version a release wrote; 1 and 2 only ever came from development builds. */ export const JOURNAL_DB_OLDEST_RELEASED_VERSION = 3 diff --git a/src/main/native-chat/agent-session-journal/journal-database.ts b/src/main/native-chat/agent-session-journal/journal-database.ts index 88a76d3b58da..8209eaf71950 100644 --- a/src/main/native-chat/agent-session-journal/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-database.ts @@ -11,11 +11,12 @@ import { createAgentSessionRecordTablesSql, createJournalTablesSql, JOURNAL_DB_OLDEST_RELEASED_VERSION, + JOURNAL_DB_RECORDS_VERSION, JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' import { JournalUnreleasedSchemaError } from './journal-open-failure' import { ensureQueuedMessagesTable } from './queued-message-schema' -import { ensureJournalSessionStateTable } from './journal-session-state' +import { createJournalSessionStatusTable } from './journal-session-state' export const JOURNAL_BUSY_TIMEOUT_MS = 5000 /** Bounds the WAL a checkpoint leaves behind; SQLite truncates it back to this after a reset. */ @@ -54,7 +55,7 @@ export function journalPragmaNumber(db: Database.Database, name: string): number export function journalDatabaseMigratesRecords(stored: number): boolean { return ( stored === 0 || - (stored >= JOURNAL_DB_OLDEST_RELEASED_VERSION && stored < JOURNAL_DB_SCHEMA_VERSION) + (stored >= JOURNAL_DB_OLDEST_RELEASED_VERSION && stored < JOURNAL_DB_RECORDS_VERSION) ) } @@ -88,7 +89,7 @@ export function journalDatabaseHoldsAgentSessions(dbPath: string): boolean | und (table) => tables.has(table) && db.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() !== undefined ) - if (holds || journalPragmaNumber(db, 'user_version') >= JOURNAL_DB_SCHEMA_VERSION) { + if (holds || journalPragmaNumber(db, 'user_version') >= JOURNAL_DB_RECORDS_VERSION) { return holds } return undefined @@ -129,10 +130,8 @@ export function openJournalDatabase( const legacyRecordImportOwed = migrateJournalSchema(probe, stored, legacyRecords) // Outside `migrateJournalSchema` on purpose: its early return skips a db // already at the current version, and this table must exist at EVERY - // writable open with no `user_version` bump (see `ensureQueuedMessagesTable`). So must each - // chat's stored state. + // writable open with no `user_version` bump (see `ensureQueuedMessagesTable`). ensureQueuedMessagesTable(probe) - ensureJournalSessionStateTable(probe) hardenSqliteDatabaseFiles(dbPath) transferred = true return { db: probe, readOnly: false, legacyRecordImportOwed } @@ -165,6 +164,8 @@ function configureJournalPragmas(db: Database.Database, stored: number): void { * Table creation, the records copy and the `user_version` bump are ONE transaction. Creating the * tables first left a shaped database still reporting version 0, which an older build does not * latch read-only; and "copied" is `user_version >= 4`, so no other marker can disagree with it. + * Version 5 adds each chat's status table, recreated empty on every run: while the records copy is + * owed the database stays at a released version an older build can write, so no row survives one. * Returns whether the copy is still owed. */ function migrateJournalSchema( @@ -179,15 +180,18 @@ function migrateJournalSchema( if (stored === 0) { db.exec(createJournalTablesSql()) } - db.exec(createAgentSessionRecordTablesSql()) - if (legacyRecords.owed) { - // A fresh file still takes a released version, so an older build opens it as one. - if (stored === 0) { - db.pragma(`user_version = ${JOURNAL_DB_OLDEST_RELEASED_VERSION}`) + createJournalSessionStatusTable(db) + if (stored < JOURNAL_DB_RECORDS_VERSION) { + db.exec(createAgentSessionRecordTablesSql()) + if (legacyRecords.owed) { + // A fresh file still takes a released version, so an older build opens it as one. + if (stored === 0) { + db.pragma(`user_version = ${JOURNAL_DB_OLDEST_RELEASED_VERSION}`) + } + return } - return + legacyRecords.write(db) } - legacyRecords.write(db) db.pragma(`user_version = ${JOURNAL_DB_SCHEMA_VERSION}`) }) return legacyRecords.owed diff --git a/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts b/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts index 89722959799d..7e0504f835c5 100644 --- a/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-dispatch-reducer.ts @@ -15,11 +15,14 @@ export function applyJournalDispatchRow( state: JournalReducerState, row: Extract ): void { - const submission = state.submissions.get(row.clientMessageId) + const current = state.submissions.get(row.clientMessageId) // Shared with the queued-draft returned hook: a row ignored here must not alter a draft. - if (!submission || !journalDispatchRowApplies(submission)) { + if (!current || !journalDispatchRowApplies(current)) { return } + // Replaced, never changed in place: a holder of the old object keeps what had committed. + const submission = { ...current } + state.submissions.set(row.clientMessageId, submission) submission.fence = row.fence submission.dispatchState = row.state submission.providerItemId = row.providerItemId diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index 52a5478c381a..39f095f1d18e 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -4,7 +4,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import { replaceJournalEpoch, type JournalReplacementItem } from './journal-epoch-replacement' -import { publishNewEpoch } from './journal-epoch-rollover' +import { publishNewEpoch, type JournalEpochStateWriter } from './journal-epoch-rollover' import type { JournalLoad } from './journal-open' import type { AgentJournalEpochReason } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' @@ -22,6 +22,8 @@ export class JournalEpochController { highestFence: () => number cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void + /** The chat's status for the new epoch, inside its transaction. */ + writeState: JournalEpochStateWriter } ) {} @@ -33,6 +35,7 @@ export class JournalEpochController { reason, fence, now: this.deps.now(), + writeState: this.deps.writeState, onPublished: this.deps.adopt }) } @@ -69,6 +72,7 @@ export class JournalEpochController { items, now: this.deps.now, mintEpoch: this.deps.mintEpoch, + writeState: this.deps.writeState, onPublished: this.deps.adopt }) return this.deps.cursor() diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts index 646254b4932d..63bccce5582d 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts @@ -69,6 +69,7 @@ function replace(input: { items: input.items, now, mintEpoch: () => `epoch-${clock}`, + writeState: () => undefined, onPublished: input.onPublished ?? (() => undefined) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 8d94c5b2b5b0..961947589788 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -1,8 +1,9 @@ // Republishing a live item set into a fresh epoch. // // One transaction: discard the old epoch's rows, insert the epoch row plus the -// replacement items, move the session projection, and retire any repair marker -// — this republished history is exactly what the marker was holding out for. +// replacement items, move the session projection, retire any repair marker +// — this republished history is exactly what the marker was holding out for — +// and write the chat's status for the new epoch. import type { AgentJournalItemBody, @@ -14,6 +15,7 @@ import type { import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' import { clearJournalRepairMarker } from './journal-repair-marker' +import type { JournalEpochStateWriter } from './journal-epoch-rollover' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { buildJournalItemRow, journalRowBase } from './journal-row-builders' import { @@ -42,6 +44,7 @@ export function replaceJournalEpoch(input: { items: readonly JournalReplacementItem[] now: () => number mintEpoch: () => string + writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -82,6 +85,7 @@ export function replaceJournalEpoch(input: { insertJournalRow(db, sessionId, row) } publishJournalSessionEpoch(db, input.identity, epoch) + input.writeState(db, state, false) }) // COMMIT landed: on disk the superseded rows are gone and this epoch is the diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts index b7eb16b81a1d..1068a7ce7a59 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-rollover.ts @@ -1,16 +1,22 @@ // Opening a new epoch. // // One transaction: discard every row of the superseded epoch, insert the new -// epoch row at sequence 1, move the session projection onto it, and retire any -// repair marker the superseded epoch was carrying. Superseded rows are DELETED -// rather than retained — nothing would ever shed them. +// epoch row at sequence 1, move the session projection onto it, retire any +// repair marker the superseded epoch was carrying, and write the chat's status for +// the new epoch. Superseded rows are DELETED rather than retained — nothing would +// ever shed them. import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import type { JournalLoad } from './journal-open' import { clearJournalRepairMarker } from './journal-repair-marker' -import { applyJournalRow, createJournalReducerState } from './journal-reducer' +import type Database from '../../sqlite/sync-database' +import { + applyJournalRow, + createJournalReducerState, + type JournalReducerState +} from './journal-reducer' import { deleteJournalEpochRows, insertJournalRow, @@ -19,6 +25,14 @@ import { } from './journal-row-table' import type { AgentJournalEpochReason, JournalRow } from './journal-row-schema' +/** Writes the chat's status from the fold an epoch transaction is about to publish, and whether that + * load reads back corrupt. A throw fails the transaction. */ +export type JournalEpochStateWriter = ( + db: Database.Database, + state: JournalReducerState, + corrupt: boolean +) => void + export function publishNewEpoch(input: { database: JournalHostDatabase identity: AgentSessionJournalIdentity @@ -26,6 +40,7 @@ export function publishNewEpoch(input: { reason: AgentJournalEpochReason fence: number now: number + writeState: JournalEpochStateWriter /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void }): void { @@ -42,6 +57,12 @@ export function publishNewEpoch(input: { } const { sessionId } = input.identity + // Folded first, so the transaction stores the status of the epoch it publishes. A repair's epoch + // reads back corrupt until the chat writes past it, as its replay would. + const state = createJournalReducerState(sessionId, input.epoch) + applyJournalRow(state, row) + state.oldestSequence = 1 + const corrupt = input.reason === 'unreconcilable_prefix' input.database.transaction((db) => { const retired = readJournalSessionEpoch(db, sessionId) if (retired !== null) { @@ -50,19 +71,11 @@ export function publishNewEpoch(input: { clearJournalRepairMarker(db, sessionId) insertJournalRow(db, sessionId, row) publishJournalSessionEpoch(db, input.identity, input.epoch) + input.writeState(db, state, corrupt) }) // COMMIT landed: on disk the superseded prefix is gone and this epoch is the // live one. The caller adopts that immediately, or a later failure leaves the // store writing into an epoch that no longer exists. - const state = createJournalReducerState(sessionId, input.epoch) - applyJournalRow(state, row) - state.oldestSequence = 1 - // A repair's epoch reads back corrupt until the chat writes past it, as its replay would. - input.onPublished({ - state, - readOnly: false, - corrupt: input.reason === 'unreconcilable_prefix', - malformedRows: 0 - }) + input.onPublished({ state, readOnly: false, corrupt, malformedRows: 0 }) } diff --git a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts index dbe15b4709ed..427f251bc29c 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts @@ -7,6 +7,7 @@ import { NO_LEGACY_JOURNAL_RECORDS } from './journal-database' import { JournalHostDatabase } from './journal-host-database' import { replayJournal, type JournalLoad } from './journal-open' import { serializeJournalRow, type JournalRow } from './journal-row-schema' +import { readJournalSessionStatuses, type JournalSessionStatus } from './journal-session-state' import { iterateJournalEpochRows, publishJournalSessionEpoch, @@ -80,6 +81,17 @@ export function createTrackedJournalOpener(): TrackedJournalOpener { } } +/** The chat's stored status, or null when it has none. */ +export function readTestJournalSessionStatus( + stateDirectory: string, + sessionId: string +): JournalSessionStatus | null { + const [stored] = readJournalSessionStatuses(openTestJournalHostDatabase(stateDirectory).db, [ + sessionId + ]) + return stored?.status ?? null +} + /** What a fresh open of the chat would replay, read from the test state directory's database. */ export function loadTestJournal(stateDirectory: string, sessionId: string): JournalLoad | null { return replayJournal(openTestJournalHostDatabase(stateDirectory).db, sessionId) diff --git a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts index 1826f18fab9d..058e64316bea 100644 --- a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts +++ b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts @@ -1,9 +1,9 @@ // What an open owes a chat whose earlier host process is gone, one rule per kind of entity. // -// The open's settlement plan (`structured-agent-session-open-settlement.ts`) and the stored -// "owes work" flag (`journal-session-state.ts`) both read these same rules, so the flag says a chat -// owes work exactly when its open would write something. Each rule revises its entity out of the -// state that selected it, so once the plan commits the flag reads false again. +// The settlement plan (`structured-agent-session-open-settlement.ts`) and each chat's stored status +// (`journal-session-state.ts`) both read these same rules, so the status shows exactly the work a +// settle would revise. Each rule revises its entity out of the state that selected it, so once the +// plan commits the stored status reads settled. import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { @@ -53,86 +53,51 @@ export function owesRecoveredDispatch(submission: AgentJournalSubmission): boole ) } -/** The writer fence of an `unverifiable` turn, which only death evidence naming it revises. */ -export function unverifiableTurnOwnerFence( - item: Pick, - itemFence: (itemId: string) => number | undefined -): number | undefined { - return readAgentJournalTurn(item.body)?.state === 'unverifiable' - ? itemFence(item.itemId) - : undefined +/** What a chat's settle would have to revise, counted from the same rules the settle applies. */ +export type JournalSettlementFacts = { + /** A turn record or tool call still `running`, under a key a settle can revise. */ + runningWork: boolean + /** A pending approval or question, under a key a settle can revise. */ + pendingPrompts: boolean + /** Handed-over sends nothing answered (`owesRecoveredDispatch`). */ + handedOverSends: number + /** Sends accepted and never handed over. */ + queuedSends: number + /** A working subagent roster or live background-task row; not counted on a corrupt load, whose + * settle leaves rosters for the rebuild. */ + liveChildWork: boolean } -export type JournalOwedFacts = { - /** The open writes something whatever the record says. */ - owesWork: boolean - /** Distinct writer fences of `unverifiable` turns, sorted. */ - unverifiableOwnerFences: number[] -} - -/** The plan's rules read without building rows. Rosters are skipped on a corrupt load, as the - * open skips them: that journal still owes a rebuild, and a write would retire the demand. */ -export function owesOpenSettlement( - fold: Pick, +export function journalSettlementFacts( + fold: Pick, options: { settlesRosters: boolean } -): JournalOwedFacts { - let owesWork = false +): JournalSettlementFacts { + const facts: JournalSettlementFacts = { + runningWork: false, + pendingPrompts: false, + handedOverSends: 0, + queuedSends: 0, + liveChildWork: false + } for (const submission of fold.submissions.values()) { - // A queued one is a leftover: its process is gone, since a close abandons what it queued. - if (owesRecoveredDispatch(submission) || isQueuedAgentJournalSubmission(submission)) { - owesWork = true - break + if (isQueuedAgentJournalSubmission(submission)) { + facts.queuedSends += 1 + } else if (owesRecoveredDispatch(submission)) { + facts.handedOverSends += 1 } } - const fences = new Set() - const itemFence = (itemId: string) => fold.itemFences.get(itemId) for (const item of fold.items.values()) { - const owed = itemOwes(item, itemFence) - owesWork ||= owed.settles || (options.settlesRosters && owed.roster) - if (owed.unverifiableOwnerFence !== undefined) { - fences.add(owed.unverifiableOwnerFence) + facts.liveChildWork ||= options.settlesRosters && staleSubagentRosterRevision(item) !== null + const running = isRunningJournalTurn(item) || isRunningToolCall(item) + const prompt = !running && openSettlementTerminalBody(item) !== null + if ((running || prompt) && openSettlementItemIdentity(item)) { + facts.runningWork ||= running + facts.pendingPrompts ||= prompt } } - return { owesWork, unverifiableOwnerFences: [...fences].sort((a, b) => a - b) } + return facts } -type ItemOwes = { settles: boolean; roster: boolean; unverifiableOwnerFence: number | undefined } - -// Per item object: a write replaces the object it revises, and an item's creating fence never -// changes, so each append judges only the items it touched. -const ITEM_OWES = new WeakMap() - -function itemOwes( - item: AgentJournalRenderItem, - itemFence: (itemId: string) => number | undefined -): ItemOwes { - let owes = ITEM_OWES.get(item) - if (!owes) { - const fence = unverifiableTurnOwnerFence(item, itemFence) - const revisable = - (fence !== undefined || - isRunningJournalTurn(item) || - openSettlementTerminalBody(item) !== null) && - openSettlementItemIdentity(item) !== null - owes = { - settles: revisable && fence === undefined, - roster: staleSubagentRosterRevision(item) !== null, - unverifiableOwnerFence: revisable ? fence : undefined - } - ITEM_OWES.set(item, owes) - } - return owes -} - -/** The record-dependent rule, as the plan applies it: an `unverifiable` turn is revised only by - * death evidence naming its writer. An older build's evidence names no writer, so selects none. */ -export function owesOnOpen( - facts: JournalOwedFacts, - deathEvidence: { ownerFence?: number } | null | undefined -): boolean { - const ownerFence = deathEvidence?.ownerFence - return ( - facts.owesWork || - (ownerFence !== undefined && facts.unverifiableOwnerFences.includes(ownerFence)) - ) +function isRunningToolCall(item: Pick): boolean { + return item.body.kind === 'tool-call' && item.body.state === 'running' } diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts index 9f9d1b052c1d..f2a4cd21eb82 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts @@ -23,11 +23,12 @@ import { liveTestJournalRows, loadTestJournal, openTestJournalHostDatabase, - readTestJournalRows + readTestJournalRows, + readTestJournalSessionStatus } from './journal-host-database-test-support' import { journalDirectoryFor, legacyJournalDatabaseFile } from './journal-paths' import { importPerSessionJournal } from './journal-per-session-import' -import { readJournalSessionState } from './journal-session-state' +import { deriveJournalSessionStatus } from './journal-session-state' import { readJournalSessionEpoch, type JournalStoredRow } from './journal-row-table' vi.mock('node:fs', async (importOriginal) => { @@ -209,15 +210,16 @@ describe('importing a per-chat journal', () => { expect(await leftovers()).toEqual([]) }) - // T10: the stored state described rows the copy replaced, so the copy's publish drops it. - it('drops the chat stored state in the copy publish, and nothing else writes one', async () => { + // T10: the copy's publish writes the chat's status for the rows it publishes, in the same + // transaction, replacing whatever described the rows it replaced. + it('writes the chat status in the copy publish, for the rows it published', async () => { const { epoch, rows } = await historyRows() await writeLegacyJournal(epoch, rows) const { db } = openTestJournalHostDatabase(root) db.prepare( - `INSERT INTO journal_session_state (session_id, state_version, epoch, seq, owes_work, - unverifiable_owner_fences, summary_json, last_activity_at, written_at) - VALUES (?, 1, 'stale-epoch', 9, 1, NULL, NULL, 1, 1)` + `INSERT INTO journal_session_state (session_id, status, active_turn_id, handed_over_sends, + queued_sends, live_child_work, summary_json, last_activity_at) + VALUES (?, 'running', NULL, 3, 0, 0, '{"status":"working","latestPrompt":"stale"}', 1)` ).run(IDENTITY.sessionId) await importPerSessionJournal({ @@ -226,12 +228,10 @@ describe('importing a per-chat journal', () => { legacyDirectory: legacyDir() }) - expect(readJournalSessionState(db, IDENTITY.sessionId)).toBeNull() - expect( - db - .prepare('SELECT COUNT(*) AS n FROM journal_session_state WHERE session_id = ?') - .get(IDENTITY.sessionId) - ).toEqual({ n: 0 }) + const loaded = loadTestJournal(root, IDENTITY.sessionId)! + expect(readTestJournalSessionStatus(root, IDENTITY.sessionId)).toEqual( + deriveJournalSessionStatus(loaded.state, { settlesRosters: !loaded.corrupt }) + ) }) // T-B3: the upgrade restart is the restart that produced the offers. A new epoch or renumbered diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts index 0c062ce97e52..901d8add652e 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts @@ -19,7 +19,7 @@ // an empty chat founded here would take a new epoch the next open's import could not reconcile. import { createHash } from 'node:crypto' -import { deleteJournalSessionState } from './journal-session-state' +import { writeJournalSessionStatusFromDisk } from './journal-session-state' import { existsSync } from 'node:fs' import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' @@ -225,8 +225,7 @@ async function copyLegacyJournal( ) } writePerSessionImportMarker(db, sessionId, legacy) - // Whatever was stored described the rows this copy replaced; the next open re-derives it. - deleteJournalSessionState(db, sessionId) + writeJournalSessionStatusFromDisk(db, sessionId) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-repair-marker.ts b/src/main/native-chat/agent-session-journal/journal-repair-marker.ts index 4a95151b96a8..d7a6de671a3d 100644 --- a/src/main/native-chat/agent-session-journal/journal-repair-marker.ts +++ b/src/main/native-chat/agent-session-journal/journal-repair-marker.ts @@ -16,7 +16,6 @@ import type Database from '../../sqlite/sync-database' import type { JournalHostDatabase } from './journal-host-database' import { deleteJournalRowSuffix } from './journal-row-table' -import { deleteJournalSessionState } from './journal-session-state' const SELECT_REPAIR = 'SELECT epoch, content_from FROM journal_repairs WHERE session_id = ?' const UPSERT_REPAIR = `INSERT INTO journal_repairs (session_id, epoch, content_from, repaired_at) @@ -57,12 +56,13 @@ export function deleteJournalRepairedSuffix(input: { /** First sequence left free once the suffix is gone. */ contentFrom: number now: number + /** The chat's status for what is left, inside the same transaction. */ + writeStatus: (db: Database.Database) => void }): number { return input.database.transaction((db) => { const deleted = deleteJournalRowSuffix(db, input.sessionId, input.epoch, input.fromSeq) db.prepare(UPSERT_REPAIR).run(input.sessionId, input.epoch, input.contentFrom, input.now) - // It described the dropped suffix; the open that follows re-derives it. - deleteJournalSessionState(db, input.sessionId) + input.writeStatus(db) return deleted }) } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts index aa0078b88a59..95d8e2d7bfda 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.test.ts @@ -68,10 +68,13 @@ describe('journal row writer', () => { readOnly: () => readOnly, highestFence: () => 0, nextSequence: () => sequence, - commit: (committed) => { + apply: () => undefined, + writeStatus: () => undefined, + committed: (committed) => { committedRows.push(committed) sequence = committed.seq + 1 - } + }, + recoverFold: () => undefined }) return { writer, committedRows } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index a29585580ebe..a1ec3f05131b 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -18,7 +18,15 @@ export type JournalRowWriterDeps = { readOnly: () => boolean highestFence: () => number nextSequence: () => number - commit: (row: JournalRow) => void + /** Folds the row inside the transaction, so the status written with it describes it. */ + apply: (row: JournalRow) => void + /** The chat's status from the fold that now holds the row. A throw fails the append. */ + writeStatus: (db: Database.Database, row: JournalRow) => void + /** After COMMIT: observers learn of the row only once it is durable. */ + committed: (row: JournalRow) => void + /** A transaction that failed after `apply`: the fold is ahead of the disk, so it is folded + * again from what committed. */ + recoverFold: () => void /** Standing hook run for EVERY appended row — the queued-draft returned * transition rides here so no rejection path can bypass it. Bookkeeping: it * runs in its own savepoint, so its failure is reported and never vetoes the row. */ @@ -40,21 +48,26 @@ export class JournalRowWriter { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) const row = build(this.deps.nextSequence(), this.deps.now()) assertJournalFence(row.fence, this.deps.highestFence()) + let applied = false try { - // One INSERT: the chat's epoch pointer moves only when the epoch does. + // One INSERT: the chat's epoch pointer moves only when the epoch does. The hook and the + // bookkeeping read the fold before the row; the status reads it after. this.deps.database().transaction((db) => { insertJournalRow(db, this.deps.sessionId, row) hook?.(db, row) this.runBookkeeping(db, row) + applied = true + this.deps.apply(row) + this.deps.writeStatus(db, row) }) } catch (error) { this.deps.rolledBack?.() + if (applied) { + this.deps.recoverFold() + } throw error } - // COMMIT landed, so the row is durable: adopt it before anything that can - // fail. Rejecting here instead would leave the next append reusing a - // sequence the table already holds. - this.deps.commit(row) + this.deps.committed(row) return row }) } diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts index 530db21d51fb..c193d26ae50e 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts @@ -173,8 +173,8 @@ export const JOURNAL_SESSION_STATE_CORPUS = { export type JournalSessionStateCase = keyof typeof JOURNAL_SESSION_STATE_CORPUS -/** Whether each case owes its open work whatever the record says: what the flag must read. */ -export const CORPUS_OWES_WORK: Record = { +/** Whether each case's stored status shows work a gone process left: what startup selects. */ +export const CORPUS_UNSETTLED: Record = { empty: false, settled: false, 'older turn running beside a newer one': true, @@ -185,7 +185,7 @@ export const CORPUS_OWES_WORK: Record = { 'pending send': true, 'unknown send': true, 'queued leftover': true, - // Owed only to death evidence naming its writer. + // Its verdict was decided when it settled; startup never revises it. 'unverifiable turn': false, 'settled roster': false } diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 07a3b9149f1d..f3544c7ad8ce 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -1,43 +1,42 @@ -// Each chat's stored state: written right after each journal commit, equal to what a fresh replay -// derives, never able to fail the write it describes, and trusted only at the chat's live tip. +// Each chat's stored status: written in the same transaction as every journal write, equal to what a +// fresh replay derives, for every chat state; a failed write leaves the fold equal to the disk. -import { createHash } from 'node:crypto' -import { isDeepStrictEqual } from 'node:util' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - AGENT_SESSION_JOURNAL_SCHEMA_VERSION, - type AgentSessionJournalIdentity -} from '../../../shared/agent-session-journal-types' -import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' import { journalPragmaNumber } from './journal-database' import { createTrackedJournalOpener, - insertTestJournalRow, insertTestJournalRowJson, loadTestJournal, - openTestJournalHostDatabase + openTestJournalHostDatabase, + readTestJournalSessionStatus } from './journal-host-database-test-support' +import * as JournalOpen from './journal-open' import { renderJournalState } from './journal-reducer' -import { deleteJournalRepairedSuffix } from './journal-repair-marker' import { - deriveJournalSessionState, - JOURNAL_SESSION_STATE_VERSION, - readJournalSessionState, - readJournalSessionStatesAtTip, - type DerivedJournalSessionState + deriveJournalSessionStatus, + isUnsettledJournalSessionStatus, + readUnsettledJournalSessionIds, + type JournalSessionStatus } from './journal-session-state' import { + CORPUS_UNSETTLED, JOURNAL_SESSION_STATE_CASES, JOURNAL_SESSION_STATE_CORPUS, type JournalSessionStateCase } from './journal-session-state-test-corpus' import type { AgentSessionJournal } from './journal-store' +vi.mock('./journal-open', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, replayJournal: vi.fn(actual.replayJournal) } +}) + const journals = createTrackedJournalOpener() let root: string let clock = 1_000 @@ -66,22 +65,15 @@ function db() { } /** What a fresh open of the chat would derive, read back from disk. */ -function freshDerivation(sessionId: string): DerivedJournalSessionState { +function freshDerivation(sessionId: string): JournalSessionStatus { const loaded = loadTestJournal(root, sessionId) if (!loaded) { throw new Error(`no journal for ${sessionId}`) } - return deriveJournalSessionState(loaded.state, { settlesRosters: !loaded.corrupt }) + return deriveJournalSessionStatus(loaded.state, { settlesRosters: !loaded.corrupt }) } -function stored(sessionId: string) { - const row = readJournalSessionState(db(), sessionId) - if (!row) { - return null - } - const { stateVersion: _stateVersion, writtenAt: _writtenAt, ...derived } = row - return derived -} +const stored = (sessionId: string) => readTestJournalSessionStatus(root, sessionId) async function write(name: JournalSessionStateCase): Promise { const journal = await open(name) @@ -89,6 +81,33 @@ async function write(name: JournalSessionStateCase): Promise { + if (failing && sql === 'COMMIT') { + failing = false + throw new Error('COMMIT failed: disk I/O error') + } + return exec(sql) + }) +} + +function refuseStatusWrites(): void { + db().exec(`CREATE TEMP TRIGGER fail_status_write BEFORE UPDATE ON main.journal_session_state + BEGIN SELECT RAISE(ABORT, 'status write refused'); END`) +} + beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-journal-session-state-')) clock = 1_000 @@ -100,31 +119,27 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }) }) -describe('the stored state follows every append (T3)', () => { +describe('the stored status follows every write (T3)', () => { it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { const journal = await open(name) const mismatches: string[] = [] - // Observers hear of a commit first; its state lands in the same task, before the next write. + // Told after each commit, which carried the row and its status together. journal.observeCommits(() => { - queueMicrotask(() => { - const expected = freshDerivation(name) - if (!isDeepStrictEqual(stored(name), expected)) { - mismatches.push(`seq ${expected.seq}`) - } - }) + const expected = freshDerivation(name) + try { + expect(stored(name)).toEqual(expected) + } catch { + mismatches.push(`seq ${journal.cursor().sequence}`) + } }) await JOURNAL_SESSION_STATE_CORPUS[name](journal) - await Promise.resolve() expect(mismatches).toEqual([]) expect(stored(name)).toEqual(freshDerivation(name)) + expect(isUnsettledJournalSessionStatus(stored(name)!)).toBe(CORPUS_UNSETTLED[name]) + expect(readUnsettledJournalSessionIds(db()).includes(name)).toBe(CORPUS_UNSETTLED[name]) }) }) -function refuseStateWrites(): void { - db().exec(`CREATE TEMP TRIGGER fail_state_write BEFORE UPDATE ON main.journal_session_state - BEGIN SELECT RAISE(ABORT, 'state write refused'); END`) -} - describe('a chat that opened corrupt stores what a fresh replay derives (T3)', () => { const name = 'working subagent roster' @@ -136,25 +151,21 @@ describe('a chat that opened corrupt stores what a fresh replay derives (T3)', ( insertTestJournalRowJson(db(), name, tip.sequence + 1, '{"not a row"') const reopened = await open(name) expect(reopened.needsRebuild).toBe(true) - // Rosters wait for the rebuild, as the open's plan does. + // The repair wrote the status; rosters wait for the rebuild, as the settle does. expect(stored(name)).toEqual(freshDerivation(name)) - expect(stored(name)).toMatchObject({ owesWork: false }) + expect(stored(name)).toMatchObject({ liveChildWork: false }) return reopened } - it('owes the roster again once the chat writes past the repair', async () => { + it('shows the roster again once the chat writes past the repair', async () => { const journal = await openCorrupt() - await journal.appendItem( - { provider: 'orca', clientMessageId: 'note-1' }, - { kind: 'status', text: 'a note' }, - { fence: 3, turnScope: { kind: 'thread' } } - ) + await note(journal, 'note-1') expect(journal.needsRebuild).toBe(false) expect(stored(name)).toEqual(freshDerivation(name)) - expect(stored(name)).toMatchObject({ owesWork: true }) + expect(stored(name)).toMatchObject({ liveChildWork: true }) }) - it('owes the roster again once provider history rebuilds the chat', async () => { + it('shows the roster again once provider history rebuilds the chat', async () => { const journal = await openCorrupt() await journal.replaceEpochItems('legacy_import', 3, [ { @@ -174,144 +185,99 @@ describe('a chat that opened corrupt stores what a fresh replay derives (T3)', ( ]) expect(journal.needsRebuild).toBe(false) expect(stored(name)).toEqual(freshDerivation(name)) - expect(stored(name)).toMatchObject({ owesWork: true }) + expect(stored(name)).toMatchObject({ liveChildWork: true }) }) }) -describe('a failing state write never fails the append (T2)', () => { - it('commits the row, leaves the state behind the tip, and the next open re-derives it', async () => { +describe('a failed write leaves the fold equal to the disk (T1)', () => { + it('folds nothing, stores nothing, and the next append takes the sequence, after a failed COMMIT', async () => { const journal = await write('settled') + const tip = journal.cursor() const before = stored('settled') - const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) - refuseStateWrites() + failNextCommit() - await expect( - journal.appendItem( - { provider: 'orca', clientMessageId: 'note-1' }, - { kind: 'status', text: 'a note' }, - { fence: 3, turnScope: { kind: 'thread' } } - ) - ).resolves.toMatchObject({ cursor: { sequence: before!.seq + 1 } }) - - expect(loadTestJournal(root, 'settled')?.state.lastSequence).toBe(before!.seq + 1) - expect(stored('settled')).toEqual(before) - expect(readJournalSessionStatesAtTip(db(), ['settled'])).toEqual([ - { sessionId: 'settled', current: null } - ]) - expect(warn).toHaveBeenCalledWith( - '[agent-session-journal] storing the chat state failed', - expect.objectContaining({ sessionId: 'settled', error: 'state write refused' }) - ) + await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') - db().exec('DROP TRIGGER fail_state_write') - await journal.close() - const reopened = await open('settled') - reopened.ensureSessionState() + expect(journal.cursor()).toEqual(tip) + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) + expect(stored('settled')).toEqual(before) + await expect(note(journal, 'kept')).resolves.toMatchObject({ + cursor: { sequence: tip.sequence + 1 } + }) expect(stored('settled')).toEqual(freshDerivation('settled')) - expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).not.toBeNull() }) -}) -describe('a failed COMMIT leaves the fold equal to the disk (T1)', () => { - function note(journal: AgentSessionJournal, id: string) { - return journal.appendItem( - { provider: 'orca', clientMessageId: id }, - { kind: 'status', text: id }, - { fence: 3, turnScope: { kind: 'thread' } } - ) - } - - it('folds nothing, stores nothing, and the next append takes the sequence', async () => { + it('fails the append when its status write fails: the row and its status land together or not at all', async () => { const journal = await write('settled') const tip = journal.cursor() - const before = stored('settled') - const connection = db() - const exec = connection.exec.bind(connection) - let failCommit = true - vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { - if (failCommit && sql === 'COMMIT') { - failCommit = false - throw new Error('COMMIT failed: disk I/O error') - } - return exec(sql) - }) + refuseStatusWrites() - await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') + await expect(note(journal, 'refused')).rejects.toThrow('status write refused') + expect(loadTestJournal(root, 'settled')?.state.lastSequence).toBe(tip.sequence) expect(journal.cursor()).toEqual(tip) expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) - expect(stored('settled')).toEqual(before) + db().exec('DROP TRIGGER fail_status_write') await expect(note(journal, 'kept')).resolves.toMatchObject({ cursor: { sequence: tip.sequence + 1 } }) - expect(stored('settled')).toEqual(freshDerivation('settled')) }) -}) -describe('trusted only at the live tip (T9)', () => { - it('leaves the schema version alone, so an older build stays writable', async () => { - await write('settled') - expect(journalPragmaNumber(db(), 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) + it('never shows a held submission the change that rolled back', async () => { + const journal = await open('held') + await journal.appendSubmission({ + clientMessageId: 'send-1', + payloadFingerprint: 'fp', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 3 + }) + const held = journal.submission('send-1')! + expect(held.dispatchState).toBe('pending') + failNextCommit() + + await expect( + journal.resolveDispatch({ + clientMessageId: 'send-1', + state: 'rejected', + reason: 'refused', + fence: 3 + }) + ).rejects.toThrow('COMMIT failed') + + expect(held.dispatchState).toBe('pending') + expect(journal.submission('send-1')).toMatchObject({ dispatchState: 'pending' }) }) - it("reads an older build's append as stale, and the next open re-derives it", async () => { + it('re-reads a fold whose re-read failed before its next use, and keeps the chat open', async () => { const journal = await write('settled') const tip = journal.cursor() - await journal.close() - // What an older build's append leaves: the row, and no state beside it. - insertTestJournalRow(db(), 'settled', { - v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, - kind: 'item', - itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'older-build-note' }), - revision: 1, - body: { kind: 'status', text: 'from an older build' }, - epoch: tip.epoch, - seq: tip.sequence + 1, - fence: 3, - ts: 5_000 + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.mocked(JournalOpen.replayJournal).mockImplementationOnce(() => { + throw new Error('disk I/O error') }) - expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toBeNull() + failNextCommit() - const reopened = await open('settled') - reopened.ensureSessionState() - expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toMatchObject({ - seq: tip.sequence + 1, - lastActivityAt: 5_000 - }) - }) + await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') - it("reads an older build's repair back to the same sequence as stale", async () => { - const journal = await write('settled') - const tip = journal.cursor() - const row = readJournalSessionState(db(), 'settled')! - // An older build repaired the chat after this row was written, then appended back to `seq`. - db() - .prepare( - 'INSERT INTO journal_repairs (session_id, epoch, content_from, repaired_at) VALUES (?, ?, ?, ?)' - ) - .run('settled', tip.epoch, tip.sequence, row.writtenAt + 1) - expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toBeNull() - - journal.ensureSessionState() - expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).not.toBeNull() + // The next read re-reads the disk instead of serving the fold that held the lost row. + expect(journal.cursor()).toEqual(tip) + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) + await expect(note(journal, 'kept')).resolves.toMatchObject({ + cursor: { sequence: tip.sequence + 1 } + }) }) }) -describe('the epoch and history transactions keep it (T10)', () => { - it('writes it for a new epoch, a roll and a replacement, after their transactions', async () => { +describe('epoch writes carry the status (T10)', () => { + it('writes it for a new epoch, a roll and a replacement, in their transactions', async () => { const journal = await open('epochs') - expect(stored('epochs')).toMatchObject({ - epoch: journal.epoch, - seq: 1, - owesWork: false, - summary: { status: null, latestPrompt: '' } - }) + expect(stored('epochs')).toMatchObject({ status: 'idle', summary: { status: null } }) await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) - expect(stored('epochs')).toMatchObject({ owesWork: true, summary: null }) + expect(stored('epochs')).toMatchObject({ status: 'running' }) await journal.rollEpoch('unreconcilable_prefix', 3) expect(stored('epochs')).toEqual(freshDerivation('epochs')) - expect(stored('epochs')).toMatchObject({ epoch: journal.epoch, seq: 1, owesWork: false }) + expect(stored('epochs')).toMatchObject({ status: 'idle' }) await journal.replaceEpochItems('legacy_import', 3, [ { @@ -320,83 +286,31 @@ describe('the epoch and history transactions keep it (T10)', () => { } ]) expect(stored('epochs')).toEqual(freshDerivation('epochs')) - expect(stored('epochs')).toMatchObject({ epoch: journal.epoch, seq: 2 }) }) - it('never fails a roll or a replacement when the state write fails (R1J-2)', async () => { + it('fails a roll whose status write fails, leaving the epoch where it was', async () => { const journal = await write('settled') - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - refuseStateWrites() + const epoch = journal.epoch + refuseStatusWrites() - await expect(journal.rollEpoch('handle_forked', 3)).resolves.toMatchObject({ sequence: 1 }) - expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(journal.epoch) - await expect( - journal.replaceEpochItems('handle_forked', 3, [ - { - identity: { provider: 'orca', clientMessageId: 'rebuilt-1' }, - body: { kind: 'status', text: 'rebuilt' } - } - ]) - ).resolves.toMatchObject({ sequence: 2 }) - expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(journal.epoch) - expect(readJournalSessionStatesAtTip(db(), ['settled'])[0]?.current).toBeNull() - }) + await expect(journal.rollEpoch('handle_forked', 3)).rejects.toThrow('status write refused') - it('drops it with a repair, and the next open writes it again', async () => { - const journal = await write('settled') - const tip = journal.cursor() - await journal.close() - deleteJournalRepairedSuffix({ - database: openTestJournalHostDatabase(root), - sessionId: 'settled', - epoch: tip.epoch, - fromSeq: tip.sequence, - contentFrom: tip.sequence, - now: clock + 1 - }) - expect(readJournalSessionState(db(), 'settled')).toBeNull() - - const reopened = await open('settled') - reopened.ensureSessionState() - expect(stored('settled')).toEqual(freshDerivation('settled')) + expect(journal.epoch).toBe(epoch) + expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(epoch) }) -}) - -describe('the derivation is versioned (T11)', () => { - // Pinned with the version: a change to what a row holds must bump the version, so rows an - // earlier derivation wrote read as absent and are derived again. - const GOLDEN = { - version: 2, - digest: '33a151ec037c24cbd1bd74cfaacc0bea419d132e1c4878724e416294635bba9b' - } - it('matches the digest pinned beside its version', async () => { - const derived: Record = {} - for (const name of JOURNAL_SESSION_STATE_CASES) { - await write(name) - const { epoch: _epoch, ...row } = freshDerivation(name) - derived[name] = row - } - const digest = createHash('sha256').update(JSON.stringify(derived)).digest('hex') - expect({ version: JOURNAL_SESSION_STATE_VERSION, digest }).toEqual(GOLDEN) - }) - - it('reads a row of another version as absent', async () => { + it('arrives with schema version 5', async () => { await write('settled') - db().prepare('UPDATE journal_session_state SET state_version = 99').run() - expect(readJournalSessionState(db(), 'settled')).toBeNull() - expect(readJournalSessionStatesAtTip(db(), ['settled'])).toEqual([ - { sessionId: 'settled', current: null } - ]) + expect(JOURNAL_DB_SCHEMA_VERSION).toBe(5) + expect(journalPragmaNumber(db(), 'user_version')).toBe(5) }) }) -describe('a stored summary is fence-independent (T15a, regression guard)', () => { +describe('a stored summary is fence-independent once settled (T15a, regression guard)', () => { it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { await write(name) const derived = freshDerivation(name) - if (derived.owesWork) { - expect(derived.summary).toBeNull() + if (isUnsettledJournalSessionStatus(derived)) { return } const snapshot = renderJournalState(loadTestJournal(root, name)!.state) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index 79a19130b7dd..fd6b757e5874 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -1,85 +1,83 @@ -// Each chat's lifecycle state, stored beside its journal. +// Each chat's status, stored beside its journal. // -// One row per chat: whether its next open owes settlement work (see -// journal-open-settlement-plan.ts), and the status summary a session list shows. Written right -// after the journal commit it describes, in its own transaction, and trusted only while its -// `(epoch, seq)` is the chat's live tip, so a write that failed, was lost, or was never made (an -// older build's append) reads as stale and the chat is re-derived. Deleting a row is always safe for -// the same reason. +// One row per chat, written in the SAME transaction as every journal write that changes the chat, +// from the fold that write produces: whether work is running or a prompt is waiting, which sends +// were handed over and never answered or are still queued, live child work, and the status summary +// a session list shows. A failed status write fails the write it describes, so a row is always the +// chat's current state. Startup selects the chats a gone process left with work from these rows, +// and seeds every other chat's status from them. // -// Created at every writable open with no `user_version` bump, as the draft table is -// (queued-message-schema.ts): an older build ignores it and stays writable. +// The table arrives with schema version 5, so a build older than that opens the database read-only +// and never writes a journal row without its status. import type Database from '../../sqlite/sync-database' import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' +import { activeStructuredAgentSessionTurnIdBySequence } from '../../../shared/structured-agent-session-live-turn' import { projectStructuredAgentSessionStatusState, type StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' -import { owesOpenSettlement, type JournalOwedFacts } from './journal-open-settlement-plan' +import { journalSettlementFacts } from './journal-open-settlement-plan' +import { replayJournal } from './journal-open' import { renderJournalState, type JournalReducerState } from './journal-reducer' -/** Version of the rules that derive a row. A row of another version reads as absent. Bump it with - * any change to what `deriveJournalSessionState` produces; the golden test pins it. */ -export const JOURNAL_SESSION_STATE_VERSION = 2 +/** Work a gone process can have left: running work, a waiting prompt, unanswered or queued sends, + * or live child work. Startup settles exactly these chats. */ +const UNSETTLED = `status <> 'idle' OR handed_over_sends > 0 OR queued_sends > 0 OR live_child_work = 1` -export function ensureJournalSessionStateTable(db: Database.Database): void { +/** Recreated, empty, by every migration up to version 5: any row it held may predate writes an + * older build made, and an open writes a missing row back. */ +export function createJournalSessionStatusTable(db: Database.Database): void { db.exec(` -CREATE TABLE IF NOT EXISTS journal_session_state ( - session_id TEXT PRIMARY KEY, - state_version INTEGER NOT NULL, - epoch TEXT NOT NULL, - seq INTEGER NOT NULL, - owes_work INTEGER NOT NULL, - unverifiable_owner_fences TEXT, - summary_json TEXT, - last_activity_at INTEGER NOT NULL, - written_at INTEGER NOT NULL +DROP TABLE IF EXISTS journal_session_state; +CREATE TABLE journal_session_state ( + session_id TEXT PRIMARY KEY, + status TEXT NOT NULL, + active_turn_id TEXT, + handed_over_sends INTEGER NOT NULL, + queued_sends INTEGER NOT NULL, + live_child_work INTEGER NOT NULL, + summary_json TEXT NOT NULL, + last_activity_at INTEGER NOT NULL ); -CREATE INDEX IF NOT EXISTS journal_session_state_owed - ON journal_session_state (session_id) - WHERE owes_work = 1 OR unverifiable_owner_fences IS NOT NULL; +CREATE INDEX journal_session_state_unsettled ON journal_session_state (session_id) + WHERE ${UNSETTLED}; `) } -export type DerivedJournalSessionState = JournalOwedFacts & { - epoch: string - seq: number - /** Only while nothing is owed: an owing chat is settled and republished by its open. */ - summary: StructuredAgentSessionStatusProjection | null +export type JournalSessionStatus = { + /** `running` covers a starting turn too: the journal records no separate start. */ + status: 'idle' | 'running' | 'attention' + activeTurnId: string | null + handedOverSends: number + queuedSends: number + liveChildWork: boolean + summary: StructuredAgentSessionStatusProjection lastActivityAt: number } -export type StoredJournalSessionState = DerivedJournalSessionState & { - stateVersion: number - writtenAt: number -} - -export type JournalSessionStateInput = { - /** False while the chat's load is corrupt: its open settles no roster either. */ +export type JournalSessionStatusInput = { + /** False while the chat's load is corrupt: its settle leaves rosters for the rebuild. */ settlesRosters: boolean currentFence?: number - /** The fold's status summary, when the caller already projected this tip. */ + /** The fold's status summary, when the caller already projects this tip. */ statusSummary?: () => StructuredAgentSessionStatusProjection } -export function deriveJournalSessionState( +export function deriveJournalSessionStatus( state: JournalReducerState, - input: JournalSessionStateInput -): DerivedJournalSessionState { - const facts = owesOpenSettlement(state, input) - let summary: StructuredAgentSessionStatusProjection | null = null - if (!facts.owesWork) { - // Fence-independent here: nothing unanswered or queued is left for the fence to judge. - summary = input.statusSummary - ? input.statusSummary() - : projectSummary(state, input.currentFence) - } + input: JournalSessionStatusInput +): JournalSessionStatus { + const facts = journalSettlementFacts(state, input) return { - ...facts, - epoch: state.epoch, - seq: state.lastSequence, - summary, + status: facts.runningWork ? 'running' : facts.pendingPrompts ? 'attention' : 'idle', + activeTurnId: activeStructuredAgentSessionTurnIdBySequence(state.items.values()), + handedOverSends: facts.handedOverSends, + queuedSends: facts.queuedSends, + liveChildWork: facts.liveChildWork, + summary: input.statusSummary + ? input.statusSummary() + : projectSummary(state, input.currentFence), lastActivityAt: state.lastActivityAt } } @@ -93,154 +91,130 @@ function projectSummary( .summary } -const UPSERT_STATE = `INSERT INTO journal_session_state (session_id, state_version, epoch, seq, - owes_work, unverifiable_owner_fences, summary_json, last_activity_at, written_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +export function isUnsettledJournalSessionStatus(status: JournalSessionStatus): boolean { + return ( + status.status !== 'idle' || + status.handedOverSends > 0 || + status.queuedSends > 0 || + status.liveChildWork + ) +} + +const UPSERT_STATUS = `INSERT INTO journal_session_state (session_id, status, active_turn_id, + handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET - state_version = excluded.state_version, epoch = excluded.epoch, seq = excluded.seq, - owes_work = excluded.owes_work, unverifiable_owner_fences = excluded.unverifiable_owner_fences, - summary_json = excluded.summary_json, last_activity_at = excluded.last_activity_at, - written_at = excluded.written_at` -const DELETE_STATE = 'DELETE FROM journal_session_state WHERE session_id = ?' -const STATE_COLUMNS = `st.state_version AS state_version, st.epoch AS state_epoch, st.seq AS seq, - st.owes_work AS owes_work, st.unverifiable_owner_fences AS unverifiable_owner_fences, - st.summary_json AS summary_json, st.last_activity_at AS last_activity_at, - st.written_at AS written_at` -const SELECT_STATE = `SELECT ${STATE_COLUMNS} FROM journal_session_state st WHERE st.session_id = ?` + status = excluded.status, active_turn_id = excluded.active_turn_id, + handed_over_sends = excluded.handed_over_sends, queued_sends = excluded.queued_sends, + live_child_work = excluded.live_child_work, summary_json = excluded.summary_json, + last_activity_at = excluded.last_activity_at` +const STATUS_COLUMNS = `st.status AS status, st.active_turn_id AS active_turn_id, + st.handed_over_sends AS handed_over_sends, st.queued_sends AS queued_sends, + st.live_child_work AS live_child_work, st.summary_json AS summary_json, + st.last_activity_at AS last_activity_at` -export function writeJournalSessionState( +/** Inside the transaction that wrote the rows it describes. */ +export function writeJournalSessionStatus( db: Database.Database, sessionId: string, - derived: DerivedJournalSessionState, - writtenAt: number + status: JournalSessionStatus ): void { - db.prepare(UPSERT_STATE).run( + db.prepare(UPSERT_STATUS).run( sessionId, - JOURNAL_SESSION_STATE_VERSION, - derived.epoch, - derived.seq, - derived.owesWork ? 1 : 0, - derived.unverifiableOwnerFences.length > 0 - ? JSON.stringify(derived.unverifiableOwnerFences) - : null, - derived.summary ? JSON.stringify(derived.summary) : null, - derived.lastActivityAt, - writtenAt + status.status, + status.activeTurnId, + status.handedOverSends, + status.queuedSends, + status.liveChildWork ? 1 : 0, + JSON.stringify(status.summary), + status.lastActivityAt ) } -export function deleteJournalSessionState(db: Database.Database, sessionId: string): void { - db.prepare(DELETE_STATE).run(sessionId) +/** For a write that publishes rows it did not fold (a per-chat file's copy, a repair): the status + * of what a replay of them reads, written in the same transaction. */ +export function writeJournalSessionStatusFromDisk(db: Database.Database, sessionId: string): void { + const loaded = replayJournal(db, sessionId) + if (loaded) { + writeJournalSessionStatus( + db, + sessionId, + deriveJournalSessionStatus(loaded.state, { settlesRosters: !loaded.corrupt }) + ) + } } -/** This build's row for the chat, or null when there is none or another version wrote it. */ -export function readJournalSessionState( - db: Database.Database, - sessionId: string -): StoredJournalSessionState | null { - const row = db.prepare(SELECT_STATE).get(sessionId) - return row ? parseStoredState(row) : null +/** Whether the chat has a status row: an open writes one for a chat an older build last wrote. */ +export function hasJournalSessionStatus(db: Database.Database, sessionId: string): boolean { + return ( + db.prepare('SELECT 1 FROM journal_session_state WHERE session_id = ?').get(sessionId) !== + undefined + ) } -/** A chat's stored state beside its live tip, for the startup seed. */ -export type JournalSessionStateAtTip = { +/** A chat's stored status for the startup seed: null when the chat has a journal but no row. */ +export type StoredJournalSessionStatus = { sessionId: string - /** Null when absent, of another version, behind the tip, or older than a repair. */ - current: StoredJournalSessionState | null + status: JournalSessionStatus | null } // Bounded well under SQLite's host-parameter limit. const IN_LIST_CHUNK = 500 /** One query per chunk of ids; ids with no journal yet are not returned. */ -export function readJournalSessionStatesAtTip( +export function readJournalSessionStatuses( db: Database.Database, sessionIds: readonly string[] -): JournalSessionStateAtTip[] { - const results: JournalSessionStateAtTip[] = [] +): StoredJournalSessionStatus[] { + const results: StoredJournalSessionStatus[] = [] for (let start = 0; start < sessionIds.length; start += IN_LIST_CHUNK) { const chunk = sessionIds.slice(start, start + IN_LIST_CHUNK) const rows = db .prepare( - `SELECT s.session_id AS session_id, s.epoch AS live_epoch, - (SELECT MAX(r.seq) FROM journal_rows r - WHERE r.session_id = s.session_id AND r.epoch = s.epoch) AS tip, - (SELECT p.repaired_at FROM journal_repairs p - WHERE p.session_id = s.session_id AND p.epoch = s.epoch) AS repaired_at, - ${STATE_COLUMNS} + `SELECT s.session_id AS session_id, ${STATUS_COLUMNS} FROM journal_sessions s LEFT JOIN journal_session_state st ON st.session_id = s.session_id WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` ) .all(...chunk) for (const row of rows) { - if (typeof row.session_id !== 'string') { - continue + if (typeof row.session_id === 'string') { + results.push({ sessionId: row.session_id, status: parseStatusRow(row) }) } - const stored = parseStoredState(row) - const atTip = - stored !== null && - stored.epoch === row.live_epoch && - stored.seq === row.tip && - (typeof row.repaired_at !== 'number' || row.repaired_at <= stored.writtenAt) - results.push({ sessionId: row.session_id, current: atTip ? stored : null }) } } return results } -/** - * Every chat this build's rows say may owe work at open, through the partial index, at any position: - * a row one write behind its tip (a write that failed, or a kill between the two commits) still - * owes what it owed, since a turn running then is still running, and an open that finds nothing to - * settle appends nothing. A row that owes nothing is never selected, stale or not, so no boot opens - * a settled chat: its own open re-derives it. - */ -export function readOwedJournalSessionStates( - db: Database.Database -): (JournalOwedFacts & { sessionId: string })[] { +/** Every chat whose stored status says a gone process left it work, through the partial index. */ +export function readUnsettledJournalSessionIds(db: Database.Database): string[] { return db - .prepare( - `SELECT session_id, owes_work, unverifiable_owner_fences FROM journal_session_state -WHERE (owes_work = 1 OR unverifiable_owner_fences IS NOT NULL) AND state_version = ?` - ) - .all(JOURNAL_SESSION_STATE_VERSION) - .flatMap((row) => - typeof row.session_id === 'string' - ? [ - { - sessionId: row.session_id, - owesWork: row.owes_work === 1, - unverifiableOwnerFences: parseFences(row.unverifiable_owner_fences) - } - ] - : [] - ) + .prepare(`SELECT session_id FROM journal_session_state WHERE ${UNSETTLED}`) + .all() + .flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) } -function parseStoredState(row: Record): StoredJournalSessionState | null { +const STATUSES = ['idle', 'running', 'attention'] as const + +function parseStatusRow(row: Record): JournalSessionStatus | null { + const status = STATUSES.find((known) => known === row.status) + const summary = parseSummary(row.summary_json) if ( - row.state_version !== JOURNAL_SESSION_STATE_VERSION || - typeof row.state_epoch !== 'string' || - typeof row.seq !== 'number' || - typeof row.last_activity_at !== 'number' || - typeof row.written_at !== 'number' + !status || + !summary || + typeof row.handed_over_sends !== 'number' || + typeof row.queued_sends !== 'number' || + typeof row.last_activity_at !== 'number' ) { return null } - const owesWork = row.owes_work === 1 - const summary = owesWork ? null : parseSummary(row.summary_json) - if (!owesWork && !summary) { - return null - } return { - stateVersion: JOURNAL_SESSION_STATE_VERSION, - epoch: row.state_epoch, - seq: row.seq, - owesWork, - unverifiableOwnerFences: parseFences(row.unverifiable_owner_fences), + status, + activeTurnId: typeof row.active_turn_id === 'string' ? row.active_turn_id : null, + handedOverSends: row.handed_over_sends, + queuedSends: row.queued_sends, + liveChildWork: row.live_child_work === 1, summary, - lastActivityAt: row.last_activity_at, - writtenAt: row.written_at + lastActivityAt: row.last_activity_at } } @@ -255,27 +229,23 @@ function parseJson(value: unknown): unknown { } } -function parseFences(value: unknown): number[] { - const parsed = parseJson(value) - return Array.isArray(parsed) - ? parsed.filter((fence): fence is number => Number.isSafeInteger(fence)) - : [] -} - -const STATUSES = ['idle', 'working', 'attention'] as const +const SUMMARY_STATUSES = ['idle', 'working', 'attention'] as const -function parseStatus(value: unknown): StructuredAgentSessionStatusProjection['status'] | undefined { - return value === null ? null : STATUSES.find((status) => status === value) +function parseSummaryStatus( + value: unknown +): StructuredAgentSessionStatusProjection['status'] | undefined { + return value === null ? null : SUMMARY_STATUSES.find((status) => status === value) } -/** Known keys only: a newer writer's extra keys are ignored rather than trusted. */ +/** Known keys only. A value this build cannot read makes the row unreadable, so the chat is opened + * and publishes what its open derives rather than a row missing that field. */ function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | null { const parsed = parseJson(value) if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { return null } const source = new Map(Object.entries(parsed)) - const status = parseStatus(source.get('status')) + const status = parseSummaryStatus(source.get('status')) const latestPrompt = source.get('latestPrompt') if (status === undefined || typeof latestPrompt !== 'string') { return null @@ -286,8 +256,6 @@ function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | } const turnOutcome = source.get('turnOutcome') if (turnOutcome !== undefined && !isAgentTurnOutcome(turnOutcome)) { - // A verdict this build cannot show: stale, so the chat's open publishes it rather than a - // row without it. return null } const statusStartedAt = source.get('statusStartedAt') @@ -304,22 +272,3 @@ function parseSummary(value: unknown): StructuredAgentSessionStatusProjection | ...(typeof statusStartedAt === 'number' ? { statusStartedAt } : {}) } } - -/** - * The open's re-derive: rewrites the chat's row when it is absent, another version's, off the - * fold's tip, or older than a repair. What an older build, an import, a repair or a failed or lost - * write left behind is caught here. Bookkeeping: the caller logs a failure and goes on. - */ -export function ensureJournalSessionStateCurrent( - db: Database.Database, - state: JournalReducerState, - input: JournalSessionStateInput, - now: number -): boolean { - const [stored] = readJournalSessionStatesAtTip(db, [state.sessionId]) - if (stored?.current?.epoch === state.epoch && stored.current.seq === state.lastSequence) { - return false - } - writeJournalSessionState(db, state.sessionId, deriveJournalSessionState(state, input), now) - return true -} diff --git a/src/main/native-chat/agent-session-journal/journal-status-projection.ts b/src/main/native-chat/agent-session-journal/journal-status-projection.ts index 9423a9bfe7ae..282921fc2d6f 100644 --- a/src/main/native-chat/agent-session-journal/journal-status-projection.ts +++ b/src/main/native-chat/agent-session-journal/journal-status-projection.ts @@ -28,12 +28,6 @@ export class JournalStatusProjection { return tip && tip.fence === fence ? tip.state : this.project(fence) } - /** Whatever fence it was projected under: a chat that owes nothing projects the same summary - * under every fence (nothing unanswered is left for the fence to judge). */ - summaryAtTip(fence: number | undefined): JournalStatusProjectionState['summary'] { - return (this.atTip()?.state ?? this.project(fence)).summary - } - private atTip(): Cached | null { const fold = this.fold() const cached = this.cached diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 918bce2cd575..2e9f5ead9b49 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -18,11 +18,16 @@ import { JournalRowWriter } from './journal-row-writer' import { restoreJournalStore } from './journal-store-restore' import type { JournalRow } from './journal-row-schema' import type { AgentSessionJournal } from './journal-store' +import type Database from '../../sqlite/sync-database' +import { applyJournalRow } from './journal-reducer' +import { readJournalRowsAfterCursor, replayJournal } from './journal-open' +import { readJournalSince } from './journal-cursor' +import type { JournalReadSince } from './journal-store-contracts' import { JOURNAL_REPAIR_DISCLOSURE_ITEM_ID } from './journal-repair-disclosure' import { - deriveJournalSessionState, - ensureJournalSessionStateCurrent, - writeJournalSessionState + deriveJournalSessionStatus, + hasJournalSessionStatus, + writeJournalSessionStatus } from './journal-session-state' import { JournalStatusProjection } from './journal-status-projection' @@ -47,10 +52,12 @@ export type JournalStoreHost = { setReadOnly: (readOnly: boolean) => void cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void - commit: (row: JournalRow) => void + /** The fold re-read from disk after a failed append. */ + replaceState: (state: JournalReducerState) => void + /** That re-read failed too: the fold is re-read before its next use, never served as it is. */ + markFoldStale: () => void /** Whether a fresh replay would report the history corrupt. */ loadCorrupt: () => boolean - /** Records whether the loaded history is corrupt. */ setLoadCorrupt: (corrupt: boolean) => void /** The conversation's fence, which the stored status reads as the status feed does. */ currentFence: () => number | undefined @@ -71,15 +78,30 @@ export type JournalStoreCollaborators = { /** Restores the store's state from disk. Owned here because it needs the same * collaborators the constructor just built. */ restore: () => Promise - /** Rewrites the chat's stored state if it does not describe the fold. */ - ensureSessionState: () => void + /** Writes the chat's status if it has none: a chat an older build last wrote. */ + backfillSessionStatus: () => void + readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince statusProjection: JournalStatusProjection } export function createJournalStoreCollaborators(host: JournalStoreHost): JournalStoreCollaborators { const statusProjection = new JournalStatusProjection(host.state) - const storeState = (onlyIfStale: boolean) => - storeJournalSessionState(host, statusProjection, onlyIfStale) + const writeStatus = (db: Database.Database, state: JournalReducerState, corrupt: boolean) => + writeJournalSessionStatus( + db, + host.identity.sessionId, + deriveJournalSessionStatus(state, { + settlesRosters: !corrupt, + currentFence: host.currentFence(), + // The live fold's projection, shared with the status feed; an epoch's new fold has its own. + ...(state === host.state() + ? { statusSummary: () => statusProjection.at(host.currentFence()).summary } + : {}) + }) + ) + // Any row but the repair's own disclosure retires the rebuild a repair owed, as replay reads it. + const corruptAfter = (row: JournalRow) => + host.loadCorrupt() && row.kind === 'item' && row.itemId === JOURNAL_REPAIR_DISCLOSURE_ITEM_ID const epochController = new JournalEpochController({ identity: host.identity, now: host.now, @@ -93,8 +115,8 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal adopt: (loaded) => { host.setLoadCorrupt(loaded.corrupt) host.adopt(loaded) - storeState(false) - } + }, + writeState: writeStatus }) const queuedMessages = new JournalQueuedMessages({ sessionId: host.identity.sessionId, @@ -109,7 +131,25 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal epochController, queuedMessages, statusProjection, - ensureSessionState: () => storeState(true), + backfillSessionStatus: () => backfillSessionStatus(host, writeStatus), + // Here rather than on the store, which is at its length limit. + readSince: (cursor, limit) => + readJournalSince( + { + state: host.state(), + rowsAfter: (afterSequence) => + readJournalRowsAfterCursor( + host.database().db, + host.identity.sessionId, + host.state().epoch, + afterSequence, + limit + ), + readOnly: host.readOnly() + }, + cursor, + host.cursor + ), // Behind the stored fact: settles drafts whose consumed submission the loaded journal shows // refused (a downgrade wrote no hook), then prunes. Bookkeeping, never failing the open. restore: () => @@ -124,14 +164,13 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal readOnly: host.readOnly, highestFence: () => host.state().highestFence, nextSequence: () => host.state().lastSequence + 1, - commit: (row) => { - // Any row but the repair's own disclosure retires the rebuild a repair owed, as replay reads it. - if (!(row.kind === 'item' && row.itemId === JOURNAL_REPAIR_DISCLOSURE_ITEM_ID)) { - host.setLoadCorrupt(false) - } - host.commit(row) - storeState(false) + apply: (row) => applyJournalRow(host.state(), row), + writeStatus: (db, row) => writeStatus(db, host.state(), corruptAfter(row)), + committed: (row) => { + host.setLoadCorrupt(corruptAfter(row)) + host.notifyCommitted() }, + recoverFold: () => recoverJournalFold(host), // Every rejection is a dispatch row through this one writer; the draft // returned-transition rides it so no path can bypass the hook. inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), @@ -149,38 +188,47 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal } } -/** - * The chat's stored state for the fold a commit (or an open) left, in its own transaction after the - * journal's, unsynced: bookkeeping that never fails the write it describes. A failed or lost write - * leaves the row behind the tip, which reads as stale, and the next open or startup re-derives it. - */ -function storeJournalSessionState( +/** Bookkeeping: a failure leaves the chat without a row, which its next open writes again. */ +function backfillSessionStatus( host: JournalStoreHost, - projection: JournalStatusProjection, - onlyIfStale: boolean + writeStatus: (db: Database.Database, state: JournalReducerState, corrupt: boolean) => void ): void { const database = host.database() if (host.readOnly() || database.readOnly || host.importPending()) { return } - const input = { - settlesRosters: !host.loadCorrupt(), - currentFence: host.currentFence(), - statusSummary: () => projection.summaryAtTip(host.currentFence()) - } try { - database.unsyncedTransaction((db) => { - if (onlyIfStale) { - ensureJournalSessionStateCurrent(db, host.state(), input, host.now()) - } else { - const state = deriveJournalSessionState(host.state(), input) - writeJournalSessionState(db, host.identity.sessionId, state, host.now()) + database.transaction((db) => { + if (!hasJournalSessionStatus(db, host.identity.sessionId)) { + writeStatus(db, host.state(), host.loadCorrupt()) } }) } catch (error) { - console.warn('[agent-session-journal] storing the chat state failed', { + console.warn('[agent-session-journal] writing a chat status failed', { + sessionId: host.identity.sessionId, + error + }) + } +} + +/** + * An append whose transaction failed after its row was folded: the fold is ahead of the disk, and + * an in-place apply cannot be undone, so the chat is folded again from what committed. The host + * database rolls a stranded transaction back before it hands out the connection, so the re-read + * never sees the row that failed. A re-read that fails leaves the fold marked stale. + */ +function recoverJournalFold(host: JournalStoreHost): void { + try { + const reloaded = replayJournal(host.database().db, host.identity.sessionId) + if (reloaded) { + host.replaceState(reloaded.state) + return + } + } catch (error) { + console.warn('[agent-session-journal] re-reading a chat after a failed append failed', { sessionId: host.identity.sessionId, - error: error instanceof Error ? error.message : String(error) + error }) } + host.markFoldStale() } diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index cd118c7ee78e..fc1426ca692c 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -13,6 +13,7 @@ import { createJournalReducerState } from './journal-reducer' import type { JournalStoreHost } from './journal-store-collaborators' import { openJournalStoreState } from './journal-store-open' import { deleteJournalRepairedSuffix } from './journal-repair-marker' +import { writeJournalSessionStatusFromDisk } from './journal-session-state' import { importPerSessionJournal, previewPerSessionJournal } from './journal-per-session-import' import { AgentSessionJournalError } from './journal-write-guards' @@ -57,7 +58,8 @@ export async function restoreJournalStore( epoch: host.state().epoch, fromSeq, contentFrom, - now: host.now() + now: host.now(), + writeStatus: (db) => writeJournalSessionStatusFromDisk(db, host.identity.sessionId) }), start: () => collaborators.epochController.start('session_created', 0), // `unreconcilable_prefix` is the durable statement that this epoch exists diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 4523a98ad543..5064a1e87a65 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -25,16 +25,14 @@ import { newestStructuredAgentSessionTurnBySequence } from '../../../shared/structured-agent-session-live-turn' import type { JournalReplacementItem } from './journal-epoch-replacement' -import { readJournalSince } from './journal-cursor' import type { JournalHostDatabase } from './journal-host-database' -import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open' +import { replayJournal, type JournalLoad } from './journal-open' import { markJournalPendingSubmissionsUnknown, rejectJournalPendingSubmissions, rejectJournalQueuedSubmissions } from './journal-pending-submission-recovery' import { - applyJournalRow, createJournalReducerState, renderJournalState, resolveJournalItemId, @@ -79,7 +77,9 @@ export class AgentSessionJournal { private readonly now: () => number private readonly mintEpoch: () => string - private state: JournalReducerState + private fold: JournalReducerState + /** A failed append's re-read from disk failed too: the fold may hold a row that never committed. */ + private foldStale = false private readOnly = false private malformedRows = 0 /** A fresh replay would report the history corrupt: it is still owed a rebuild. */ @@ -92,9 +92,10 @@ export class AgentSessionJournal { private readonly itemAppender: JournalItemAppender private readonly lifecycleBatchAppender: JournalLifecycleBatchAppender private readonly restore: () => Promise - /** Rewrites the chat's stored state if it does not describe this fold. Bookkeeping: never + /** Writes the chat's status if it has none (an older build last wrote it). Bookkeeping: never * fails the open that calls it. */ - readonly ensureSessionState: () => void + readonly backfillSessionStatus: () => void + readonly readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince private readonly statusProjection: JournalStatusProjection /** Draft rows queued while the agent works; never reducer input or owed work. */ readonly queuedMessages: JournalQueuedMessages @@ -104,7 +105,7 @@ export class AgentSessionJournal { this.database = options.database this.now = options.now ?? (() => Date.now()) this.mintEpoch = options.mintEpoch ?? randomUUID - this.state = createJournalReducerState(options.identity.sessionId, '') + this.fold = createJournalReducerState(options.identity.sessionId, '') // Serializes sequence assignment with the durable write behind it. this.queue = new JournalWriteQueue(options.identity.sessionId) const collaborators = createJournalStoreCollaborators({ @@ -126,9 +127,11 @@ export class AgentSessionJournal { this.adoptLoadedJournal(loaded) this.onCommitted?.() }, - commit: (row) => { - applyJournalRow(this.state, row) - this.onCommitted?.() + replaceState: (state) => { + this.state = state + }, + markFoldStale: () => { + this.foldStale = true }, loadCorrupt: () => this.loadCorrupt, currentFence: options.currentFence ?? (() => undefined), @@ -150,10 +153,28 @@ export class AgentSessionJournal { this.lifecycleBatchAppender = collaborators.lifecycleBatchAppender this.queuedMessages = collaborators.queuedMessages this.restore = collaborators.restore - this.ensureSessionState = collaborators.ensureSessionState + this.backfillSessionStatus = collaborators.backfillSessionStatus + this.readSince = collaborators.readSince this.statusProjection = collaborators.statusProjection } + /** Re-read from disk first while stale, so no reader or writer gets a fold the disk never held. */ + private get state(): JournalReducerState { + if (this.foldStale) { + const reloaded = replayJournal(this.database.db, this.identity.sessionId) + if (!reloaded) { + throw new Error(`agent-session journal ${this.identity.sessionId} is gone`) + } + this.state = reloaded.state + } + return this.fold + } + + private set state(state: JournalReducerState) { + this.fold = state + this.foldStale = false + } + get isReadOnly(): boolean { return this.readOnly } @@ -289,25 +310,6 @@ export class AgentSessionJournal { canonicalItemId = (itemId: string): string => resolveJournalItemId(this.state, itemId) - readSince(cursor: AgentJournalCursor, limit?: number): JournalReadSince { - return readJournalSince( - { - state: this.state, - rowsAfter: (afterSequence) => - readJournalRowsAfterCursor( - this.database.db, - this.identity.sessionId, - this.state.epoch, - afterSequence, - limit - ), - readOnly: this.readOnly - }, - cursor, - () => this.cursor() - ) - } - /** Upsert by stable identity. The revision is assigned here so a caller * cannot accidentally publish a revision the reducer will drop. */ appendItem( diff --git a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts index 2c9c6cfcd040..d1600c4fd994 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts @@ -78,12 +78,15 @@ export function acceptSubmissionFromProviderItem( if (providerItemId === resolvedItemId) { return } - const submission = [...state.submissions.values()].find( + const current = [...state.submissions.values()].find( (candidate) => agentJournalSubmissionKey(candidate.clientMessageId) === resolvedItemId ) - if (!submission || !journalDispatchRowApplies(submission)) { + if (!current || !journalDispatchRowApplies(current)) { return } + // Replaced, never changed in place, as the dispatch reducer does. + const submission = { ...current } + state.submissions.set(current.clientMessageId, submission) submission.fence = row.fence submission.dispatchState = 'accepted' notePersonTurnAccepted(state, submission) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 69c8b4937fd1..6c1c55f34a44 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -103,6 +103,11 @@ export function createStructuredAgentSessionConversationLifetime(host: { * drop it after. */ conversation: async (sessionId: string): Promise => { + // Startup's settle first: no command acts on a chat a gone process left mid-turn. + const startup = deps().commandsReady?.() + if (startup) { + await startup + } const open = sessions.get(sessionId) if (open) { // Restore left its per-chat file uncopied; a reader gets the chat from the one database. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index 54420122f1e9..23bab6f221d1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -117,7 +117,7 @@ export async function openStructuredAgentSessionConversationJournal( await appendOpenSettlement(opened.journal, plan, fence, (error) => deps.onEventSinkError?.({ sessionId, error }) ) - opened.journal.ensureSessionState() + opened.journal.backfillSessionStatus() return { session: { journal: opened.journal, params, child: null }, reset: opened.recovery?.reset ?? null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index cdb7b473ae23..95040b7ba206 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -138,8 +138,15 @@ export function createStructuredAgentSessionConversationDelivery(input: { loop.dispose() stopResettling() }, - open: (sessionId, options) => - openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) + open: async (sessionId, options) => { + // Every command that opens a chat waits for startup's settle, as `conversation()` does. + await deps.commandsReady?.() + return openStructuredAgentSessionConversation( + { deps, sessions, adoptOpened }, + sessionId, + options + ) + } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index c2ec18332882..8aa6790ced91 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -122,6 +122,9 @@ export type StructuredAgentSessionHostDeps = { /** Whether an orchestration dispatch still owns this session's worker; absent answers no. */ hasOpenDispatch?: (record: AgentSessionRecord) => boolean onEventSinkError?: (input: { sessionId: string; error: unknown }) => void + /** Pending while host startup settles the chats a gone process left with work: every chat + * command waits for it, and none is refused. Null or absent once open. */ + commandsReady?: () => Promise | null /** Lease bookkeeping run for startup or a read (the reconcile, or resolving a chat's recovery) * that refused or threw, once per distinct failure. Startup and the read carry on: the next * attach or send reconciles and resolves recovery again before it acts. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts index f2e16ff7aaee..dd340cd1c144 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -1,6 +1,7 @@ -// The stored "owes work" flag and the open's settlement plan are two readings of one set of rules: -// the flag selects a chat exactly when its open would write something, the open writes exactly the -// plan, and once the plan commits the flag reads false again, so nothing is selected twice. +// A chat's stored status and its settlement plan are two readings of one set of rules: startup +// selects a chat exactly when its settle would write something, the settle writes exactly the plan, +// and once the plan commits the status reads settled, so nothing is selected twice. Death evidence +// changes only which verdict a running turn gets, never whether the chat is selected. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -16,14 +17,14 @@ import { insertTestJournalRow, insertTestJournalRowJson, liveTestJournalRows, - openTestJournalHostDatabase + openTestJournalHostDatabase, + readTestJournalSessionStatus } from '../agent-session-journal/journal-host-database-test-support' -import { owesOnOpen } from '../agent-session-journal/journal-open-settlement-plan' -import { readJournalSessionState } from '../agent-session-journal/journal-session-state' +import { isUnsettledJournalSessionStatus } from '../agent-session-journal/journal-session-state' import { CORPUS_DEATH_EVIDENCE, CORPUS_FENCE, - CORPUS_OWES_WORK, + CORPUS_UNSETTLED, JOURNAL_SESSION_STATE_CASES, JOURNAL_SESSION_STATE_CORPUS } from '../agent-session-journal/journal-session-state-test-corpus' @@ -56,14 +57,16 @@ function open(sessionId: string): Promise { }) } -function storedFacts(sessionId: string) { - const row = readJournalSessionState(openTestJournalHostDatabase(root).db, sessionId) +function storedStatus(sessionId: string) { + const row = readTestJournalSessionStatus(root, sessionId) if (!row) { - throw new Error(`no stored state for ${sessionId}`) + throw new Error(`no stored status for ${sessionId}`) } return row } +const selected = (sessionId: string) => isUnsettledJournalSessionStatus(storedStatus(sessionId)) + beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-open-settlement-')) clock = 1_000 @@ -78,7 +81,7 @@ const COMBINATIONS = JOURNAL_SESSION_STATE_CASES.flatMap((name) => Object.keys(CORPUS_DEATH_EVIDENCE).map((evidence) => [name, evidence] as const) ) -describe('the stored flag and the plan agree (T4)', () => { +describe('the stored status and the plan agree (T4)', () => { it.each(COMBINATIONS)('%s, death evidence %s', async (name, evidenceName) => { const sessionId = `chat-${(chats += 1)}` const journal = await open(sessionId) @@ -86,11 +89,14 @@ describe('the stored flag and the plan agree (T4)', () => { const deathEvidence = CORPUS_DEATH_EVIDENCE[evidenceName] ?? null const record: OpenSettlementRecordFacts = { sessionId, fence: CORPUS_FENCE, deathEvidence } - // Pinned per case, so agreement between the flag and the plan cannot hide both being wrong. - expect(storedFacts(sessionId).owesWork).toBe(CORPUS_OWES_WORK[name]) + // Pinned per case, so agreement between the status and the plan cannot hide both being wrong. + expect(selected(sessionId)).toBe(CORPUS_UNSETTLED[name]) const plan = planOpenSettlement(journal, record, { settlesRosters: true }) - const selected = owesOnOpen(storedFacts(sessionId), deathEvidence) - expect(selected).toBe(!openSettlementPlanIsEmpty(plan)) + // Evidence naming an `unverifiable` turn's writer lets an open revise it; startup never selects + // a chat for that (main's acquisition path does it when the chat is next used). + const revisesVerdictOnly = + name === 'unverifiable turn' && deathEvidence?.ownerFence === CORPUS_FENCE + expect(selected(sessionId)).toBe(!openSettlementPlanIsEmpty(plan) && !revisesVerdictOnly) // The open appends exactly the plan: a row per roster, per recovered send and per rejected // leftover, and one lifecycle batch for what the gone generation left. @@ -108,7 +114,7 @@ describe('the stored flag and the plan agree (T4)', () => { ) // Every entry revised its entity out of the state that selected it (T4b, T15b). - expect(owesOnOpen(storedFacts(sessionId), deathEvidence)).toBe(false) + expect(selected(sessionId)).toBe(false) expect( openSettlementPlanIsEmpty(planOpenSettlement(journal, record, { settlesRosters: true })) ).toBe(true) @@ -131,7 +137,7 @@ describe('the stored flag and the plan agree (T4)', () => { // The rebuild is still owed, so neither touches the roster. expect(journal.needsRebuild).toBe(true) - expect(owesOnOpen(storedFacts('corrupt'), null)).toBe(false) + expect(selected('corrupt')).toBe(false) expect(openSettlementPlanIsEmpty(plan())).toBe(true) await journal.appendItem( @@ -139,15 +145,15 @@ describe('the stored flag and the plan agree (T4)', () => { { kind: 'status', text: 'a note' }, { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } ) - expect(owesOnOpen(storedFacts('corrupt'), null)).toBe(true) + expect(selected('corrupt')).toBe(true) expect(plan().rosters).toHaveLength(1) await appendOpenSettlement(journal, plan(), CORPUS_FENCE, (error) => { throw error }) - expect(owesOnOpen(storedFacts('corrupt'), null)).toBe(false) + expect(selected('corrupt')).toBe(false) }) - it('owes nothing for an item whose key will not parse, which no plan can revise (R1J-4)', async () => { + it('selects nothing for an item whose key will not parse, which no plan can revise (R1J-4)', async () => { const first = await open('unkeyed') await JOURNAL_SESSION_STATE_CORPUS.settled(first) const tip = first.cursor() @@ -183,7 +189,8 @@ describe('the stored flag and the plan agree (T4)', () => { startedAt: 50 }) const journal = await open('unkeyed') - journal.ensureSessionState() + // Rows written beside the journal, as an older build would: the open writes the status back. + expect(readTestJournalSessionStatus(root, 'unkeyed')).not.toBeNull() const deathEvidence = CORPUS_DEATH_EVIDENCE['names the writer'] ?? null const plan = planOpenSettlement( journal, @@ -192,32 +199,21 @@ describe('the stored flag and the plan agree (T4)', () => { ) expect(openSettlementPlanIsEmpty(plan)).toBe(true) - expect(storedFacts('unkeyed')).toMatchObject({ owesWork: false, unverifiableOwnerFences: [] }) - expect(owesOnOpen(storedFacts('unkeyed'), deathEvidence)).toBe(false) - }) - - it('owes nothing for a settled chat, whatever the record says', async () => { - const journal = await open('settled') - await JOURNAL_SESSION_STATE_CORPUS.settled(journal) - for (const deathEvidence of Object.values(CORPUS_DEATH_EVIDENCE)) { - expect(owesOnOpen(storedFacts('settled'), deathEvidence)).toBe(false) - } + expect(storedStatus('unkeyed')).toMatchObject({ status: 'idle' }) + expect(selected('unkeyed')).toBe(false) }) - it("selects an unverifiable turn only by evidence naming its writer, never an older build's", async () => { + it('never selects a chat to revise a verdict: an unverifiable turn stays as it settled (D16 gone)', async () => { const journal = await open('unverifiable') await JOURNAL_SESSION_STATE_CORPUS['unverifiable turn'](journal) - const facts = storedFacts('unverifiable') - expect(facts).toMatchObject({ owesWork: false, unverifiableOwnerFences: [CORPUS_FENCE] }) - expect(owesOnOpen(facts, CORPUS_DEATH_EVIDENCE['older build, no owner'])).toBe(false) - expect(owesOnOpen(facts, CORPUS_DEATH_EVIDENCE['names another owner'])).toBe(false) - expect(owesOnOpen(facts, CORPUS_DEATH_EVIDENCE['names the writer'])).toBe(true) + expect(storedStatus('unverifiable')).toMatchObject({ status: 'idle' }) + expect(selected('unverifiable')).toBe(false) }) - it('makes a chat whose only debt is a queued leftover owe work, and settles it (T15b)', async () => { + it('selects a chat whose only debt is a queued leftover, and settles it (T15b)', async () => { const journal = await open('queued') await JOURNAL_SESSION_STATE_CORPUS['queued leftover'](journal) - expect(storedFacts('queued')).toMatchObject({ owesWork: true, summary: null }) + expect(storedStatus('queued')).toMatchObject({ status: 'idle', queuedSends: 1 }) const plan = planOpenSettlement(journal, null, { settlesRosters: true }) expect(plan.leftoverQueued).toEqual(['send-queued']) @@ -226,6 +222,6 @@ describe('the stored flag and the plan agree (T4)', () => { }) expect(journal.submission('send-queued')).toMatchObject({ dispatchState: 'rejected' }) - expect(storedFacts('queued')).toMatchObject({ owesWork: false, summary: { status: 'idle' } }) + expect(storedStatus('queued')).toMatchObject({ queuedSends: 0, summary: { status: 'idle' } }) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 57bed2d2d809..029d4aa16de3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -114,6 +114,7 @@ export function createStructuredAgentSessionHostRestore( openDeps: deps, supportsRecord, seedStatus, + resolveRecovery: readRestore.resolveRecovery, restoreListed: (records) => restoreStructuredAgentSessionsOnRestart({ ...readRestore, records }), serialize: rest.serialize, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 17f3c12f0ffb..8bb4f68169c2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -8,20 +8,20 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' -import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' -import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION } from '../../../shared/agent-session-journal-types' import { closeTestJournalHostDatabases, - insertTestJournalRow, - openTestJournalHostDatabase + openTestJournalHostDatabase, + readTestJournalSessionStatus } from '../agent-session-journal/journal-host-database-test-support' -import { readJournalSessionState } from '../agent-session-journal/journal-session-state' import Database from '../../sqlite/sync-database' +import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' +import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' import { createRestTestRig, latestRestTestStatus, restTestChat, restTestOpens, + sendRestTestMessage, type RestTestRig } from './structured-agent-session-rest-test-rig' @@ -252,50 +252,13 @@ describe('startup opens only what it must (T6, T7, T14)', () => { } await crashMidSend(rig, 'session-crashed') await crashMidSend(rig, 'session-crashed-closed', false) - await crashMidSend(rig, 'session-crashed-state-lost', false) - for (const sessionId of [ - 'session-stale', - 'session-repaired', - 'session-draft', - 'session-uncopied' - ]) { + for (const sessionId of ['session-rowless', 'session-draft', 'session-uncopied']) { await restTestChat(rig, sessionId, { message: sessionId }) } await rig.crash() - const live = (sessionId: string) => - String( - db(rig).prepare('SELECT epoch FROM journal_sessions WHERE session_id = ?').get(sessionId) - ?.epoch - ) - const tip = (sessionId: string) => readJournalSessionState(db(rig), sessionId)! - const appendPastState = (sessionId: string) => - insertTestJournalRow(db(rig), sessionId, { - v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, - kind: 'item', - itemId: agentJournalItemKey({ provider: 'orca', clientMessageId: 'unstored-note' }), - revision: 1, - body: { kind: 'status', text: 'no state stored beside it' }, - epoch: live(sessionId), - seq: tip(sessionId).seq + 1, - fence: 1, - ts: 9_000 - }) - // An older build appended past the stored state. - appendPastState('session-stale') - // A kill between a commit and its state write: the row is one behind, still owing. - appendPastState('session-crashed-state-lost') - // An older build repaired the chat after its state was stored, and wrote back to that sequence. - db(rig) - .prepare( - 'INSERT INTO journal_repairs (session_id, epoch, content_from, repaired_at) VALUES (?, ?, ?, ?)' - ) - .run( - 'session-repaired', - live('session-repaired'), - tip('session-repaired').seq, - tip('session-repaired').writtenAt + 1 - ) + // Last written before its status table existed: it has a journal and no status. + db(rig).prepare('DELETE FROM journal_session_state WHERE session_id = ?').run('session-rowless') // A draft an earlier process queued: paused by the restart, so an open would send nothing. db(rig) .prepare( @@ -322,25 +285,18 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(listed).not.toContain('session-crashed-closed') const background = await startup(rig, listed) - const unlisted = ['session-crashed-closed', 'session-crashed-state-lost'] + const unlisted = ['session-crashed-closed'] expect(opened(rig, [...listed, ...unlisted]).toSorted()).toEqual( - [ - 'session-crashed', - ...unlisted, - 'session-repaired', - 'session-stale', - 'session-uncopied' - ].toSorted() - ) - expect(background.toSorted()).toEqual( - ['session-repaired', 'session-stale', 'session-uncopied'].toSorted() + ['session-crashed', ...unlisted, 'session-rowless', 'session-uncopied'].toSorted() ) + expect(background.toSorted()).toEqual(['session-rowless', 'session-uncopied'].toSorted()) expect(latestRestTestStatus(rig, 'session-draft')).toMatchObject({ status: 'idle' }) // Settled with no user action; the listed one is open and never showed its pre-crash work. for (const sessionId of ['session-crashed', ...unlisted]) { // The unanswered send is now recovered doubt, which projects as no running request. - expect(readJournalSessionState(db(rig), sessionId)).toMatchObject({ - owesWork: false, + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ + status: 'idle', + handedOverSends: 0, summary: { status: null } }) } @@ -356,7 +312,10 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) } - // The next boot finds nothing owed and nothing stale (T4b). + // Its open wrote the missing status back. + expect(readTestJournalSessionStatus(rig.root, 'session-rowless')).not.toBeNull() + + // The next boot finds nothing to settle and nothing without a status (T4b). await rig.crash() await rig.boot() const again = await startup(rig, listedIds(rig)) @@ -364,3 +323,127 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(opened(rig, [...listedIds(rig), ...unlisted])).toEqual([]) }) }) + +describe('a provider process that outlived the crash (R1T-2)', () => { + it('is stopped at startup for a listed and an unlisted chat, and none is left on the next boot', async () => { + const rig = await newRig() + // Attached, then sent to: its provider process is live when Orca dies. + for (const [id, listed] of [ + ['session-idle-listed', true], + ['session-idle-closed', false] + ] as const) { + await restTestChat(rig, id, { listed }) + await restTestChat(rig, id, { message: 'done', listed }) + } + const ids = ['session-idle-listed', 'session-idle-closed'] + // Each chat's provider process was still up when Orca died: its lease names it, live. + const leases = ids.map((id) => rig.store.getRecord(id)!.lease) + await rig.host.flushAllStreamedEvents() + await rig.crash() + for (const lease of leases) { + await writeOlderBuildLease(rig.root, lease.sessionId, { ...lease }) + } + // The rig's chats share one recorded process; stopping it ends both. + let alive = true + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + rig.probeOwner.mockImplementation(async () => + alive + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } + ) + + await rig.boot({ stopOwnerProcess }) + await startup(rig, listedIds(rig)) + + expect(stopOwnerProcess).toHaveBeenCalled() + expect(alive).toBe(false) + for (const id of ids) { + expect(rig.store.getRecord(id)!.lease).toMatchObject({ + handoffStage: null, + ownerProcess: null, + deathEvidence: { kind: 'pid-absent' } + }) + } + // The listed chat was seeded, not opened, and still had its process stopped. + expect(opened(rig, ids)).toEqual([]) + + await rig.crash() + stopOwnerProcess.mockClear() + await rig.boot({ stopOwnerProcess }) + await startup(rig, listedIds(rig)) + expect(stopOwnerProcess).not.toHaveBeenCalled() + }) +}) + +describe('one awaited settle covers a chat whose tab closes meanwhile (R1T-4)', () => { + it('settles a listed chat the listed worker skips because its tab closed', async () => { + const rig = await newRig() + await crashMidSend(rig, 'session-closing') + await rig.crash() + await rig.boot() + const listed = listedIds(rig) + expect(listed).toContain('session-closing') + await rig.host.reconcileRestartLeases() + rig.host.seedStoredStatuses(listed) + // The tab closes after the listing named it, before its settle runs. + await rig.store.setSessionTabVisibility('session-closing', false) + + await rig.host.settleOwedSessions(listed) + + expect(readTestJournalSessionStatus(rig.root, 'session-closing')).toMatchObject({ + status: 'idle', + handedOverSends: 0 + }) + expect(rig.host.hasSession('session-closing')).toBe(false) + }) +}) + +describe('the startup gate holds chat commands and never refuses them', () => { + async function bootGated(gate: StructuredAgentSessionStartupGate): Promise { + const rig = await newRig() + await restTestChat(rig, 'session-1', { message: 'first' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + gate.hold() + await rig.boot({ commandsReady: gate.ready }) + return rig + } + + it('holds a send until the settle ends, then delivers it; listing and status answer meanwhile', async () => { + const gate = new StructuredAgentSessionStartupGate() + const rig = await bootGated(gate) + // The tab list and the seeded status answer while the gate is closed. + expect(listedIds(rig)).toContain('session-1') + rig.host.seedStoredStatuses(['session-1']) + expect(latestRestTestStatus(rig, 'session-1')).toMatchObject({ status: 'idle' }) + const dispatched = rig.adapter.dispatch.mock.calls.length + let answered = false + const sent = sendRestTestMessage(rig, 'session-1', 'during startup').then((result) => { + answered = true + return result + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(answered).toBe(false) + expect(rig.adapter.dispatch.mock.calls.length).toBe(dispatched) + + gate.openWhen(Promise.reject(new Error('settle failed'))) + + expect(await sent).toMatchObject({ ok: true }) + await vi.waitFor(() => + expect(rig.adapter.dispatch.mock.calls.length).toBeGreaterThan(dispatched) + ) + }) + + it('lets held commands through at its ceiling if the settle never ends', async () => { + const gate = new StructuredAgentSessionStartupGate(30) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const rig = await bootGated(gate) + + expect(await sendRestTestMessage(rig, 'session-1', 'after the ceiling')).toMatchObject({ + ok: true + }) + expect(gate.ready()).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 58668e49843c..eaa93db7b6f0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -1,22 +1,25 @@ -// What host startup does with the state stored beside each chat's journal, before any client -// lists a tab, and without opening a chat that owes nothing. +// What host startup does with each chat's stored status, before any client lists a tab. // -// Seed: every settled, listed chat's status row is published from its stored state, so session -// lists have it at paint. Settle: every chat whose stored state says its open owes work, listed or -// not, is opened once, one at a time, and its open appends the settlement plan. A listed one goes -// through the restart restore's own per-chat worker and stays open; an unlisted one is settled and -// closed, never indexed or published. Everything else a listed chat needs (a stale row, a per-chat -// file not yet copied) is left to the background restore after the listing, which is the same -// worker. +// Seed: every settled, listed chat's status row is published from its stored status, so session +// lists have it at paint, without opening the chat. Settle: first every lease a crash left +// `recovering` is resolved (a surviving provider process is stopped and its death recorded), so +// each verdict below is taken once, from that evidence. Then every chat whose stored status shows +// work a gone process left, listed or not, is opened once, one at a time, and its open appends the +// settlement plan. A listed one goes through the restart restore's own per-chat worker and stays +// open; any other is settled and closed, never indexed or published. Chat commands wait for the +// settle (see `StructuredAgentSessionHostDeps.commandsReady`); listing, paint and status reads do +// not. A listed chat with no stored status yet (an older build wrote it last, or a per-chat file is +// not yet copied) is left to the background restore after the listing, which is the same worker. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { owesOnOpen } from '../agent-session-journal/journal-open-settlement-plan' import { - readJournalSessionStatesAtTip, - readOwedJournalSessionStates + isUnsettledJournalSessionStatus, + readJournalSessionStatuses, + readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' import { openStructuredAgentSessionConversationJournal, @@ -24,15 +27,21 @@ import { } from './structured-agent-session-conversation-open' import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' +// Record-only (a probe and at most a process stop each), so a few at once. +const RECOVERY_CONCURRENCY = 4 + export type StructuredAgentSessionStartupStateDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { - store: Pick + store: Pick } supportsRecord: (record: AgentSessionRecord) => boolean seedStatus: ( record: AgentSessionRecord, stored: { projected: StructuredAgentSessionStatusProjection; lastActivityAt: number } ) => void + /** Resolves a `recovering` lease; never throws (a failure is reported and left to the next + * attach or send). */ + resolveRecovery: (sessionId: string) => Promise /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish). */ restoreListed: (records: AgentSessionRecord[]) => Promise serialize: (sessionId: string, task: () => Promise) => Promise @@ -43,7 +52,7 @@ export type StructuredAgentSessionStartupStateDeps = { export type StructuredAgentSessionStartupState = { /** Seeds settled listed chats; answers the listed ids the background restore still opens. */ seedStoredStatuses: (listedIds: readonly string[]) => string[] - /** Settles every chat that owes work, once per host. Never rejects. */ + /** Settles every chat a gone process left with work, once per host. Never rejects. */ settleOwedSessions: (listedIds: readonly string[]) => Promise } @@ -65,19 +74,19 @@ function seedStoredStatuses( listedIds: readonly string[] ): string[] { const database = deps.openDeps.journalDatabase - // A newer build's database: nothing is stored this build can trust, and every chat reads as it does. + // A newer build's database: nothing is stored this build can read, and every chat reads as it does. if (database.readOnly) { return [...listedIds] } - let states: ReturnType + let stored: ReturnType try { - states = readJournalSessionStatesAtTip(database.db, listedIds) + stored = readJournalSessionStatuses(database.db, listedIds) } catch (error) { - // Fails open: every listed chat is restored in the background, as before stored state existed. - console.warn('[structured-agent-session] reading stored chat state failed', error) + // Fails open: every listed chat is restored in the background, as before stored status existed. + console.warn('[structured-agent-session] reading stored chat status failed', error) return [...listedIds] } - const byId = new Map(states.map((state) => [state.sessionId, state.current])) + const byId = new Map(stored.map((row) => [row.sessionId, row.status])) const background: string[] = [] for (const sessionId of listedIds) { const record = deps.openDeps.store.getRecord(sessionId) @@ -96,18 +105,17 @@ function seedStoredStatuses( } continue } - const stored = byId.get(sessionId) - if (stored && owesOnOpen(stored, record.lease.deathEvidence)) { - // The settle step's: its open settles and publishes. + const status = byId.get(sessionId) + if (status && isUnsettledJournalSessionStatus(status)) { + // The settle's: its open settles and publishes. continue } - const status = stored?.summary?.status - if (!stored?.summary || (status !== null && status !== 'idle')) { - // Stale, or describing a state nothing owed cannot reach: re-derived by its open. + if (!status || (status.summary.status !== null && status.summary.status !== 'idle')) { + // No row this build can read: its open writes and publishes one. background.push(sessionId) continue } - deps.seedStatus(record, { projected: stored.summary, lastActivityAt: stored.lastActivityAt }) + deps.seedStatus(record, { projected: status.summary, lastActivityAt: status.lastActivityAt }) } return background } @@ -121,22 +129,19 @@ async function settleOwedSessions( if (database.readOnly) { return } + await resolveRecoveringLeases(deps) const listedOrder = new Map(listedIds.map((sessionId, index) => [sessionId, index])) const listed: AgentSessionRecord[] = [] - const unlisted: AgentSessionRecord[] = [] - for (const owed of readOwedJournalSessionStates(database.db)) { - const record = deps.openDeps.store.getRecord(owed.sessionId) - if ( - !record || - !deps.supportsRecord(record) || - !owesOnOpen(owed, record.lease.deathEvidence) - ) { + const others: AgentSessionRecord[] = [] + for (const sessionId of readUnsettledJournalSessionIds(database.db)) { + const record = deps.openDeps.store.getRecord(sessionId) + if (!record || !deps.supportsRecord(record)) { continue } - if (listedOrder.has(record.sessionId)) { + if (listedOrder.has(sessionId)) { listed.push(record) } else { - unlisted.push(record) + others.push(record) } } listed.sort( @@ -144,14 +149,16 @@ async function settleOwedSessions( (listedOrder.get(left.sessionId) ?? 0) - (listedOrder.get(right.sessionId) ?? 0) ) await deps.restoreListed(listed) - for (const record of unlisted) { + // A listed chat the worker left closed (its tab closed meanwhile) is settled like any other. + others.push(...listed.filter((record) => !deps.hasSession(record.sessionId))) + for (const record of others) { // A journal open is synchronous SQLite: one chat per macrotask. await yieldToEventLoop() if (deps.isDisposed()) { return } await deps - .serialize(record.sessionId, () => settleUnlisted(deps, record)) + .serialize(record.sessionId, () => settleClosed(deps, record)) .catch((error: unknown) => { console.warn('[structured-agent-session] settling a chat at startup failed', { sessionId: record.sessionId, @@ -164,8 +171,19 @@ async function settleOwedSessions( } } -/** A chat with no tab: settled and closed, never indexed, so it gets no status row. */ -async function settleUnlisted( +/** Every lease a crash left `recovering`, listed or not: a provider process that outlived the crash + * is stopped and its death recorded, so the settle below reads one verdict per turn. */ +async function resolveRecoveringLeases(deps: StructuredAgentSessionStartupStateDeps) { + const recovering = deps.openDeps.store + .listRecords() + .filter((record) => record.lease.handoffStage === 'recovering') + await forEachWithConcurrency(recovering, RECOVERY_CONCURRENCY, async ({ sessionId }) => { + await deps.resolveRecovery(sessionId) + }) +} + +/** A chat nothing holds open: settled and closed, never indexed, so it gets no status row. */ +async function settleClosed( deps: StructuredAgentSessionStartupStateDeps, record: AgentSessionRecord ): Promise { diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index 1d383b985c15..c89fbf6bda51 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -290,7 +290,9 @@ async function startOrcadRuntime( observedStatusCapture.attach(runtime) // Why here and not on a client's first listing: a headless host seeds its chats' statuses and - // settles crashed ones itself, before any client connects. + // settles crashed ones itself, before any client connects. Chat commands a client sends meanwhile + // wait for that settle. + runtime.holdStructuredAgentSessionCommandsForStartup() void runtime.prepareStructuredAgentSessionStartupRestoration().catch((error: unknown) => { console.warn('[structured-agent-session] startup restoration failed', error) }) diff --git a/src/main/orcad/orcad-push-startup.test.ts b/src/main/orcad/orcad-push-startup.test.ts index 390346fe3018..26326708eda3 100644 --- a/src/main/orcad/orcad-push-startup.test.ts +++ b/src/main/orcad/orcad-push-startup.test.ts @@ -75,6 +75,7 @@ vi.mock('../runtime/orca-runtime', () => ({ async refreshRestoredOrchestrationAuthority() {} async reconcileLegacyWorkerTerminals() {} async prepareStructuredAgentSessionStartupRestoration() {} + holdStructuredAgentSessionCommandsForStartup() {} setMobilePushRegistrar( registrar: Parameters[0] ) { diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index 2cb0b563f88e..cdd9b169e501 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -278,21 +278,34 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR } protected async prepareStructuredAgentSessionStartupRestorationOnce(): Promise { - if (!this.hasPersistedStructuredAgentSessionStore()) { - return - } - // Durable agent records must exist before daemon inventory can be reconciled against them. - // A refused host is no host: startup goes on, and only structured requests are refused. - await ensureStructuredAgentSessionHostUnlessRefused(() => - this.ensureStructuredAgentSessionHost() - ) - await this.refreshMobileSessionPtyRecords() - const host = getStructuredAgentSessionHost() - if (host) { - this.structuredAgentSessionBackgroundRestoreIds = await runStructuredAgentSessionStartupStep( - host, - this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + // Chat commands held for startup go ahead however this ends: when the settle does, or now. + let settling = false + try { + if (!this.hasPersistedStructuredAgentSessionStore()) { + return + } + // Durable agent records must exist before daemon inventory can be reconciled against them. + // A refused host is no host: startup goes on, and only structured requests are refused. + await ensureStructuredAgentSessionHostUnlessRefused(() => + this.ensureStructuredAgentSessionHost() ) + await this.refreshMobileSessionPtyRecords() + const host = getStructuredAgentSessionHost() + if (host) { + this.structuredAgentSessionBackgroundRestoreIds = + await runStructuredAgentSessionStartupStep( + host, + this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null, + (settled) => { + settling = true + this.structuredAgentSessionStartupGate.openWhen(settled) + } + ) + } + } finally { + if (!settling) { + this.structuredAgentSessionStartupGate.open() + } } } diff --git a/src/main/runtime/orca-runtime-get-worktree-ps.ts b/src/main/runtime/orca-runtime-get-worktree-ps.ts index adcd47baad30..830299a920d9 100644 --- a/src/main/runtime/orca-runtime-get-worktree-ps.ts +++ b/src/main/runtime/orca-runtime-get-worktree-ps.ts @@ -20,6 +20,7 @@ import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' import { buildWorktreeListingPage } from './worktree-listing-host-scope' import { structuredWorkerOwesWork } from './structured-worker-custody' +import { StructuredAgentSessionStartupGate } from './structured-agent-session-startup-gate' import { resolveTuiAgentLaunchEnv } from '../../shared/tui-agent-launch-defaults' import { nativeChatShellEnvironmentPolicy } from '../../shared/native-chat-shell-environment' import { claudeStructuredPermissionModeForSettings } from '../claude/claude-structured-permission-mode' @@ -27,6 +28,14 @@ import { codexStructuredPermissionPolicyForSettings } from '../codex/codex-struc import { claudeStructuredAuthPolicyForSettings } from '../claude-accounts/claude-structured-auth-policy' export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVisibleReadProbe { + // Chat commands wait on it while startup settles chats a gone process left with work. + protected structuredAgentSessionStartupGate = new StructuredAgentSessionStartupGate() + + /** Before any request is served: chat commands wait until the startup settle ends. */ + holdStructuredAgentSessionCommandsForStartup(): void { + this.structuredAgentSessionStartupGate.hold() + } + async getWorktreePs( limit = DEFAULT_WORKTREE_PS_LIMIT, sourceDefaultsSupported = true @@ -177,7 +186,8 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVis // Read per sweep tick from the orchestration database: a worker whose dispatch is open keeps // its agent running. No database answers no. hasOpenDispatch: (record) => - structuredWorkerOwesWork(this.getOrchestrationDbIfAvailable?.() ?? null, record) + structuredWorkerOwesWork(this.getOrchestrationDbIfAvailable?.() ?? null, record), + commandsReady: () => this.structuredAgentSessionStartupGate.ready() }) } } diff --git a/src/main/runtime/structured-agent-session-record-import-install.test.ts b/src/main/runtime/structured-agent-session-record-import-install.test.ts index 551fab477c41..1d3fd09b7e87 100644 --- a/src/main/runtime/structured-agent-session-record-import-install.test.ts +++ b/src/main/runtime/structured-agent-session-record-import-install.test.ts @@ -9,6 +9,7 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' import Database from '../sqlite/sync-database' import { journalPragmaNumber } from '../native-chat/agent-session-journal/journal-database' +import { JOURNAL_DB_SCHEMA_VERSION } from '../native-chat/agent-session-journal/journal-database-schema' import { journalDatabasePath } from '../native-chat/agent-session-journal/journal-host-database' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { legacyAgentSessionStorePath } from './agent-session-record-store-file' @@ -148,7 +149,7 @@ it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( expect(relaunched.legacyRecordImportOwed()).toBe(false) expect(second.unverifiable()).toBe(false) - expect(databaseVersion()).toBe(4) + expect(databaseVersion()).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(relaunched.deps.store.getRecord(IMPORTED)).not.toBeNull() expect(relaunched.deps.store.getRecord(CREATED_WHILE_OWED)).not.toBeNull() } @@ -165,7 +166,7 @@ it('installs over a file no read will make usable, reports it once, and leaves i { kind: 'unusable', error: expect.objectContaining({ message: 'agent_session_store_corrupt' }) } ]) expect((await host()).legacyRecordImportOwed()).toBe(false) - expect(databaseVersion()).toBe(4) + expect(databaseVersion()).toBe(JOURNAL_DB_SCHEMA_VERSION) await stopStructuredAgentSessionRuntime() expect(await readFile(legacyAgentSessionStorePath(root), 'utf-8')).toBe('{ truncated') }) diff --git a/src/main/runtime/structured-agent-session-runtime.ts b/src/main/runtime/structured-agent-session-runtime.ts index 60743010d316..21641983b092 100644 --- a/src/main/runtime/structured-agent-session-runtime.ts +++ b/src/main/runtime/structured-agent-session-runtime.ts @@ -121,6 +121,8 @@ export type StructuredAgentSessionRuntimeDeps = { statusSink?: StructuredAgentSessionHostDeps['statusSink'] /** See `StructuredAgentSessionHostDeps.hasOpenDispatch`. */ hasOpenDispatch?: StructuredAgentSessionHostDeps['hasOpenDispatch'] + /** See `StructuredAgentSessionHostDeps.commandsReady`. */ + commandsReady?: StructuredAgentSessionHostDeps['commandsReady'] /** The account home a structured launch would pin right now, for catalog * reads with no session record. Absent disables the catalog surface. */ resolveAgentAccountHome?: RuntimeAgentAccountHomeResolver @@ -340,6 +342,7 @@ async function installOnJournal( ...(deps.onSessionStatusChanged ? { onSessionStatusChanged: deps.onSessionStatusChanged } : {}), ...(deps.statusSink ? { statusSink: deps.statusSink } : {}), ...(deps.hasOpenDispatch ? { hasOpenDispatch: deps.hasOpenDispatch } : {}), + ...(deps.commandsReady ? { commandsReady: deps.commandsReady } : {}), ...(await modelCatalogHostDeps({ store, deps, envResolvers })) }) setStructuredAgentSessionHost(host) diff --git a/src/main/runtime/structured-agent-session-startup-gate.ts b/src/main/runtime/structured-agent-session-startup-gate.ts new file mode 100644 index 000000000000..dee92bdff88d --- /dev/null +++ b/src/main/runtime/structured-agent-session-startup-gate.ts @@ -0,0 +1,54 @@ +// Holds every structured chat command until host startup has settled the chats a gone process left +// with work, so no command acts on a chat whose crashed turn still reads as running. Commands wait; +// none is refused. It always opens: when the settle ends, however it ends, or at a ceiling, so a +// startup fault never strands a command. + +// Far past any measured settle; only a hung step reaches it. +const STARTUP_GATE_CEILING_MS = 120_000 + +export class StructuredAgentSessionStartupGate { + private pending: Promise | null = null + private release: (() => void) | null = null + private ceiling: ReturnType | null = null + + constructor(private readonly ceilingMs = STARTUP_GATE_CEILING_MS) {} + + /** Closes the gate until it is opened; a no-op while it is closed. Before any request is served. */ + hold(): void { + if (this.pending) { + return + } + this.pending = new Promise((resolve) => { + this.release = resolve + }) + this.ceiling = setTimeout(() => { + console.warn( + '[structured-agent-session] startup settle still running; chat commands go ahead' + ) + this.open() + }, this.ceilingMs) + this.ceiling.unref?.() + } + + /** Opens once the settle ends, whether it settled or failed. */ + openWhen(settled: Promise): void { + void settled + .catch((error: unknown) => { + console.warn('[structured-agent-session] startup settle failed', error) + }) + .finally(() => this.open()) + } + + open(): void { + if (this.ceiling) { + clearTimeout(this.ceiling) + this.ceiling = null + } + this.release?.() + this.release = null + this.pending = null + } + + /** What a chat command waits for, or null when the gate is open. */ + ready = (): Promise | null => this.pending +} diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index 692dbe67d8b2..1f6df6b54555 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -5,9 +5,9 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { closeTestJournalHostDatabases, - openTestJournalHostDatabase + readTestJournalSessionStatus } from '../native-chat/agent-session-journal/journal-host-database-test-support' -import { readJournalSessionState } from '../native-chat/agent-session-journal/journal-session-state' +import { isUnsettledJournalSessionStatus } from '../native-chat/agent-session-journal/journal-session-state' import { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { @@ -18,8 +18,10 @@ import { type RestTestRig } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' import { OrcaRuntimeService } from './orca-runtime' +import type { StructuredAgentSessionStartupGate } from './structured-agent-session-startup-gate' type PrepareInternals = { + structuredAgentSessionStartupGate: StructuredAgentSessionStartupGate store: { getWorkspaceSession: () => unknown } hasPersistedStructuredAgentSessionStore(): boolean refreshMobileSessionPtyRecords(): Promise | null> @@ -39,11 +41,15 @@ afterEach(async () => { vi.restoreAllMocks() }) +function internals(runtime: OrcaRuntimeService): PrepareInternals { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. + return runtime as unknown as PrepareInternals +} + /** A restarted runtime over the rig's host, with nothing but `prepare` ever called on it. */ function restartedRuntime(): OrcaRuntimeService { const runtime = new OrcaRuntimeService() - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these members exist on the runtime; they are protected, not absent. - const internal = runtime as unknown as PrepareInternals + const internal = internals(runtime) internal.store = { getWorkspaceSession: () => null } internal.hasPersistedStructuredAgentSessionStore = () => true internal.refreshMobileSessionPtyRecords = async () => new Set() @@ -59,9 +65,41 @@ async function crashMidSend(sessionId: string, listed = true): Promise { } function owes(sessionId: string): boolean | undefined { - return readJournalSessionState(openTestJournalHostDatabase(rig.root).db, sessionId)?.owesWork + const stored = readTestJournalSessionStatus(rig.root, sessionId) + return stored ? isUnsettledJournalSessionStatus(stored) : undefined } +it('holds chat commands until the startup settle ends, and then lets them through', async () => { + await crashMidSend('session-crashed') + await rig.crash() + await rig.boot() + const runtime = restartedRuntime() + const gate = internals(runtime).structuredAgentSessionStartupGate + runtime.holdStructuredAgentSessionCommandsForStartup() + expect(gate.ready()).not.toBeNull() + + await runtime.prepareStructuredAgentSessionStartupRestoration() + await vi.waitFor(() => expect(owes('session-crashed')).toBe(false)) + + await vi.waitFor(() => expect(gate.ready()).toBeNull()) +}) + +it('opens the gate when the startup step fails, never stranding a command', async () => { + await rig.crash() + await rig.boot() + const runtime = restartedRuntime() + const gate = internals(runtime).structuredAgentSessionStartupGate + internals(runtime).ensureStructuredAgentSessionHost = async () => { + throw new Error('host refused') + } + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + runtime.holdStructuredAgentSessionCommandsForStartup() + + await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) + + expect(gate.ready()).toBeNull() +}) + it('seeds and settles on a host no client ever lists (T8)', async () => { await restTestChat(rig, 'session-settled', { message: 'done' }) await crashMidSend('session-crashed') diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index 0a19da5a0ab7..23050b8dd7e6 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -1,6 +1,6 @@ // The structured-chat step of host startup, before any client lists a tab: the restart lease check, -// then each listed chat's status seeded from its stored state, and every chat that owes work -// settled. +// then each listed chat's status seeded from its stored status, and every chat a gone process left +// with work settled. Chat commands wait for that settle; the tab list and paint do not. import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' @@ -27,7 +27,9 @@ type StructuredAgentSessionListingHost = Partial< */ export async function runStructuredAgentSessionStartupStep( host: StructuredAgentSessionStartupHost, - savedSession: WorkspaceSessionState | null + savedSession: WorkspaceSessionState | null, + /** The settle, which chat commands wait for; it never rejects. */ + onSettling: (settled: Promise) => void ): Promise { let leaseFailure: { error: unknown } | null = null try { @@ -37,8 +39,8 @@ export async function runStructuredAgentSessionStartupStep( } const listedIds = listedStructuredAgentSessionIds(host, savedSession) const background = host.seedStoredStatuses?.(listedIds) ?? listedIds - // Un-awaited: nothing waits on a crashed chat's settle, and it never rejects. - void host.settleOwedSessions?.(listedIds) + // Not awaited here: the tab list and paint never wait on it; chat commands do. + onSettling(host.settleOwedSessions?.(listedIds) ?? Promise.resolve()) if (leaseFailure) { throw leaseFailure.error } diff --git a/src/main/startup/main-process-runtime-launch-activation.test.ts b/src/main/startup/main-process-runtime-launch-activation.test.ts index 42813af450e7..be1ebdac440a 100644 --- a/src/main/startup/main-process-runtime-launch-activation.test.ts +++ b/src/main/startup/main-process-runtime-launch-activation.test.ts @@ -153,8 +153,10 @@ describe('desktop startup activation', () => { launchHooks.failBeforeWindow = false state.mainWindow = null state.isServeMode = false - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime before the mocked RPC server takes it. - state.runtime = {} as NonNullable + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime and holds chat commands before the mocked RPC server takes it. + state.runtime = { + holdStructuredAgentSessionCommandsForStartup: () => undefined + } as unknown as NonNullable // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only calls whenReady(). state.windowsShellPathHydration = { whenReady: () => Promise.resolve() diff --git a/src/main/startup/main-process-runtime-launch.ts b/src/main/startup/main-process-runtime-launch.ts index 4b77d878cc5d..4a8a6fe132bb 100644 --- a/src/main/startup/main-process-runtime-launch.ts +++ b/src/main/startup/main-process-runtime-launch.ts @@ -308,6 +308,8 @@ export async function initializeMainProcessRuntimeLaunch( return } state.serveOptions = serveOptions + // Before anything is served: chat commands wait for the startup settle started below. + runtime.holdStructuredAgentSessionCommandsForStartup() const runtimeRpc = installRuntimeRpc(runtime, serveOptions) const shellPathReady = shellPathHydration.whenReady() // Why published: the renderer's git-environment barrier must fence on the same @@ -341,9 +343,7 @@ export async function initializeMainProcessRuntimeLaunch( // client or neither sees the same state. Behind the barriers the renderer's own call awaits. void Promise.all([state.firstWindowStartupServicesReady, state.managedWslCliStartupBarrierReady]) .then(() => runtime.prepareStructuredAgentSessionStartupRestoration()) - .catch((error: unknown) => { - console.warn('[structured-agent-session] startup restoration failed', error) - }) + .catch((error: unknown) => console.warn('[structured-agent-session] startup failed', error)) app.on('activate', options.handleMacAppActivation) if (serveOptions) { await launchServeMode(runtime, runtimeRpc, serveOptions) From 9533d1fae91c407c5484666e2b68662592575640 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:16:38 -0700 Subject: [PATCH 027/107] test(native-chat): the key guard and the repair's status write each have a test that catches their removal --- .../journal-session-state.test.ts | 24 +++++++++++++++++++ ...ured-agent-session-open-settlement.test.ts | 7 +++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index f3544c7ad8ce..5939bf894804 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -18,10 +18,12 @@ import { } from './journal-host-database-test-support' import * as JournalOpen from './journal-open' import { renderJournalState } from './journal-reducer' +import { deleteJournalRepairedSuffix } from './journal-repair-marker' import { deriveJournalSessionStatus, isUnsettledJournalSessionStatus, readUnsettledJournalSessionIds, + writeJournalSessionStatusFromDisk, type JournalSessionStatus } from './journal-session-state' import { @@ -189,6 +191,28 @@ describe('a chat that opened corrupt stores what a fresh replay derives (T3)', ( }) }) +describe('a repair writes the status of what it leaves (T10)', () => { + it('drops the rejected suffix and the status that described it, in one transaction', async () => { + const journal = await write('running tool') + const tip = journal.cursor() + await journal.close() + expect(stored('running tool')).toMatchObject({ status: 'running' }) + + deleteJournalRepairedSuffix({ + database: openTestJournalHostDatabase(root), + sessionId: 'running tool', + epoch: tip.epoch, + fromSeq: tip.sequence, + contentFrom: tip.sequence, + now: clock + 1, + writeStatus: (database) => writeJournalSessionStatusFromDisk(database, 'running tool') + }) + + expect(stored('running tool')).toEqual(freshDerivation('running tool')) + expect(stored('running tool')).toMatchObject({ status: 'idle' }) + }) +}) + describe('a failed write leaves the fold equal to the disk (T1)', () => { it('folds nothing, stores nothing, and the next append takes the sequence, after a failed COMMIT', async () => { const journal = await write('settled') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts index dd340cd1c144..4966388f6a59 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -188,8 +188,13 @@ describe('the stored status and the plan agree (T4)', () => { state: 'unverifiable', startedAt: 50 }) + // Written beside the journal, as before the status table: the open writes the status back. + openTestJournalHostDatabase(root) + .db.prepare('DELETE FROM journal_session_state WHERE session_id = ?') + .run('unkeyed') const journal = await open('unkeyed') - // Rows written beside the journal, as an older build would: the open writes the status back. + // As the chat's open does after its settle. + journal.backfillSessionStatus() expect(readTestJournalSessionStatus(root, 'unkeyed')).not.toBeNull() const deathEvidence = CORPUS_DEATH_EVIDENCE['names the writer'] ?? null const plan = planOpenSettlement( From 25ee094aed5d30192d977888f8a401f4e82ee727 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:27:29 -0700 Subject: [PATCH 028/107] test(native-chat): the held-submission case resolves with a typed dispatch --- .../agent-session-journal/journal-session-state.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 5939bf894804..9b685593da88 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -262,8 +262,13 @@ describe('a failed write leaves the fold equal to the disk (T1)', () => { await expect( journal.resolveDispatch({ clientMessageId: 'send-1', - state: 'rejected', - reason: 'refused', + state: 'accepted', + providerIdentity: { + provider: 'codex', + threadId: 'thread-held', + turnId: 'turn-1', + ordinal: 1 + }, fence: 3 }) ).rejects.toThrow('COMMIT failed') From 7668014be9551f5b7eac788ca7940206ce3508dd Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:52:25 -0700 Subject: [PATCH 029/107] refactor(native-chat): a failed append puts back what its row changed in the fold The fold's containers record each change while an append's transaction is open, and a failed transaction replays them in reverse, newest first, with the fold's counters and turn scope as they stood. Entries are replaced on change, never edited, so restoring each touched key restores the fold and an entry a reader holds never shows the failed row. Re-reading the chat from disk stays as the fallback when the undo itself fails, and marking the fold stale after that. The status projection a failed row was read into is dropped with it. --- .../journal-derived-turn-scope.ts | 7 ++ .../journal-fold-undo.ts | 113 ++++++++++++++++++ .../agent-session-journal/journal-reducer.ts | 16 +-- .../journal-session-state.test.ts | 50 +++++++- .../journal-status-projection.ts | 5 + .../journal-store-collaborators.ts | 39 ++++-- 6 files changed, 212 insertions(+), 18 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-fold-undo.ts diff --git a/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts b/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts index 0fbadede3bc7..0954a42da44f 100644 --- a/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts +++ b/src/main/native-chat/agent-session-journal/journal-derived-turn-scope.ts @@ -20,6 +20,13 @@ import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' export class JournalDerivedTurnScope { private openTurnItemId: string | null = null + /** For an append's undo: the scope as it stood before the row. */ + clone(): JournalDerivedTurnScope { + const copy = new JournalDerivedTurnScope() + copy.openTurnItemId = this.openTurnItemId + return copy + } + /** The scope for a row being created now that states none. A turn record belongs to no turn. */ scopeFor(body: AgentJournalItemBody): AgentJournalTurnScope { return this.openTurnItemId === null || readAgentJournalTurn(body) diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts new file mode 100644 index 000000000000..9021a6d2da22 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts @@ -0,0 +1,113 @@ +// One append's undo of the fold. An append folds its row inside its transaction, so the status +// written with it describes it; if the transaction fails, exactly what the row touched is put back. +// Entries are replaced on change, never edited, so restoring each touched key restores the fold, +// and an entry a reader holds is never one the failed row changed. + +import type { JournalReducerState } from './journal-reducer' + +type Undo = () => void + +/** A fold container that, while a unit is open, tells it each change it is about to make. */ +export class JournalFoldMap extends Map { + onChange: ((undo: Undo) => void) | null = null + + override set(key: K, value: V): this { + this.record(key) + return super.set(key, value) + } + + override delete(key: K): boolean { + this.record(key) + return super.delete(key) + } + + private record(key: K): void { + if (!this.onChange) { + return + } + if (super.has(key)) { + const previous = super.get(key) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: `has` just answered true, so `get` returned the key's own value, `undefined` included when V allows it. + this.onChange(() => super.set(key, previous as V)) + } else { + this.onChange(() => super.delete(key)) + } + } +} + +export class JournalFoldSet extends Set { + onChange: ((undo: Undo) => void) | null = null + + override add(value: T): this { + if (this.onChange && !super.has(value)) { + this.onChange(() => super.delete(value)) + } + return super.add(value) + } + + override delete(value: T): boolean { + if (this.onChange && super.has(value)) { + this.onChange(() => super.add(value)) + } + return super.delete(value) + } +} + +export type JournalFoldUndo = { + /** COMMIT landed: the fold as it stands is adopted. */ + commit: () => void + /** The transaction failed: every change since the unit began is put back, newest first. */ + rollback: () => void +} + +/** Null when the fold's containers cannot record their changes; the caller re-reads it instead. */ +export function beginJournalFoldUndo(state: JournalReducerState): JournalFoldUndo | null { + const containers = [ + state.items, + state.itemFences, + state.tombstones, + state.submissions, + state.receipts, + state.aliases, + state.appliedSettlementIds + ] + const recording = containers.filter( + (container): container is JournalFoldMap | JournalFoldSet => + container instanceof JournalFoldMap || container instanceof JournalFoldSet + ) + if (recording.length !== containers.length) { + return null + } + const undos: Undo[] = [] + const record = (undo: Undo) => { + undos.push(undo) + } + const scalars = { + epoch: state.epoch, + lastSequence: state.lastSequence, + lastActivityAt: state.lastActivityAt, + oldestSequence: state.oldestSequence, + highestFence: state.highestFence, + latestPersonTurnSequence: state.latestPersonTurnSequence + } + const derivedTurnScope = state.derivedTurnScope.clone() + for (const container of recording) { + container.onChange = record + } + const detach = () => { + for (const container of recording) { + container.onChange = null + } + } + return { + commit: detach, + rollback: () => { + detach() + for (const undo of undos.toReversed()) { + undo() + } + Object.assign(state, scalars) + state.derivedTurnScope = derivedTurnScope + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index ed12c4a6cba8..9c1d08bb5f16 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -24,6 +24,7 @@ import { } from '../../../shared/agent-session-journal-item-key' import { structuredAgentSessionPayloadFingerprint } from '../../../shared/structured-agent-session-mutation' import { JournalDerivedTurnScope } from './journal-derived-turn-scope' +import { JournalFoldMap, JournalFoldSet } from './journal-fold-undo' import { removeJournalItem, statedOrDerivedTurnScope, upsertJournalItem } from './journal-item-fold' import { journalItemRevisionIsStale } from './journal-item-revision' import type { JournalRow } from './journal-row-schema' @@ -67,13 +68,14 @@ export function createJournalReducerState(sessionId: string, epoch: string): Jou lastActivityAt: 0, oldestSequence: 1, highestFence: 0, - items: new Map(), - itemFences: new Map(), - tombstones: new Map(), - submissions: new Map(), - receipts: new Map(), - aliases: new Map(), - appliedSettlementIds: new Set(), + // Recording containers: an append that fails puts back exactly what it changed. + items: new JournalFoldMap(), + itemFences: new JournalFoldMap(), + tombstones: new JournalFoldMap(), + submissions: new JournalFoldMap(), + receipts: new JournalFoldMap(), + aliases: new JournalFoldMap(), + appliedSettlementIds: new JournalFoldSet(), derivedTurnScope: new JournalDerivedTurnScope(), latestPersonTurnSequence: 0 } diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 9b685593da88..579cd24a34c9 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -16,6 +16,7 @@ import { openTestJournalHostDatabase, readTestJournalSessionStatus } from './journal-host-database-test-support' +import * as JournalFoldUndo from './journal-fold-undo' import * as JournalOpen from './journal-open' import { renderJournalState } from './journal-reducer' import { deleteJournalRepairedSuffix } from './journal-repair-marker' @@ -38,6 +39,10 @@ vi.mock('./journal-open', async (importOriginal) => { const actual = await importOriginal() return { ...actual, replayJournal: vi.fn(actual.replayJournal) } }) +vi.mock('./journal-fold-undo', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, beginJournalFoldUndo: vi.fn(actual.beginJournalFoldUndo) } +}) const journals = createTrackedJournalOpener() let root: string @@ -91,6 +96,14 @@ function note(journal: AgentSessionJournal, id: string) { ) } +function reply(journal: AgentSessionJournal, text: string) { + return journal.appendItem( + { provider: 'orca', clientMessageId: 'reply' }, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text }] }, + { fence: 3, turnScope: { kind: 'thread' } } + ) +} + /** Fails the next COMMIT the connection runs, once. */ function failNextCommit(): void { const connection = db() @@ -214,21 +227,26 @@ describe('a repair writes the status of what it leaves (T10)', () => { }) describe('a failed write leaves the fold equal to the disk (T1)', () => { - it('folds nothing, stores nothing, and the next append takes the sequence, after a failed COMMIT', async () => { + it('puts back what a failed COMMIT folded, stores nothing, and the next append takes the sequence', async () => { const journal = await write('settled') const tip = journal.cursor() const before = stored('settled') failNextCommit() + vi.mocked(JournalOpen.replayJournal).mockClear() - await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') + await expect(reply(journal, 'lost')).rejects.toThrow('COMMIT failed') + // Undone in memory: the chat is not read from disk again. + expect(JournalOpen.replayJournal).not.toHaveBeenCalled() expect(journal.cursor()).toEqual(tip) expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) expect(stored('settled')).toEqual(before) - await expect(note(journal, 'kept')).resolves.toMatchObject({ + await expect(reply(journal, 'kept')).resolves.toMatchObject({ cursor: { sequence: tip.sequence + 1 } }) + // Not the projection of the reply that rolled back, which held the same sequence. expect(stored('settled')).toEqual(freshDerivation('settled')) + expect(stored('settled')?.summary.lastAssistantMessage).toBe('kept') }) it('fails the append when its status write fails: the row and its status land together or not at all', async () => { @@ -277,10 +295,34 @@ describe('a failed write leaves the fold equal to the disk (T1)', () => { expect(journal.submission('send-1')).toMatchObject({ dispatchState: 'pending' }) }) - it('re-reads a fold whose re-read failed before its next use, and keeps the chat open', async () => { + it('re-reads the chat when the undo fails, and keeps the chat open', async () => { const journal = await write('settled') const tip = journal.cursor() vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.mocked(JournalFoldUndo.beginJournalFoldUndo).mockImplementationOnce(() => ({ + commit: () => undefined, + rollback: () => { + throw new Error('undo failed') + } + })) + failNextCommit() + + await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') + + expect(journal.cursor()).toEqual(tip) + expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) + }) + + it('re-reads before next use a fold whose undo and re-read both failed, and keeps the chat open', async () => { + const journal = await write('settled') + const tip = journal.cursor() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.mocked(JournalFoldUndo.beginJournalFoldUndo).mockImplementationOnce(() => ({ + commit: () => undefined, + rollback: () => { + throw new Error('undo failed') + } + })) vi.mocked(JournalOpen.replayJournal).mockImplementationOnce(() => { throw new Error('disk I/O error') }) diff --git a/src/main/native-chat/agent-session-journal/journal-status-projection.ts b/src/main/native-chat/agent-session-journal/journal-status-projection.ts index 282921fc2d6f..3830ff20c07e 100644 --- a/src/main/native-chat/agent-session-journal/journal-status-projection.ts +++ b/src/main/native-chat/agent-session-journal/journal-status-projection.ts @@ -28,6 +28,11 @@ export class JournalStatusProjection { return tip && tip.fence === fence ? tip.state : this.project(fence) } + /** A failed append's projection read a row that never committed. */ + invalidate(): void { + this.cached = null + } + private atTip(): Cached | null { const fold = this.fold() const cached = this.cached diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 2e9f5ead9b49..b74259aab3b7 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -30,6 +30,7 @@ import { writeJournalSessionStatus } from './journal-session-state' import { JournalStatusProjection } from './journal-status-projection' +import { beginJournalFoldUndo, type JournalFoldUndo } from './journal-fold-undo' export type JournalStoreHost = { /** Fires the journal's commit listener for a durable change that appended no @@ -99,6 +100,8 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal : {}) }) ) + // The open append's undo: what its row changed in the fold, put back if its transaction fails. + let undo: JournalFoldUndo | null = null // Any row but the repair's own disclosure retires the rebuild a repair owed, as replay reads it. const corruptAfter = (row: JournalRow) => host.loadCorrupt() && row.kind === 'item' && row.itemId === JOURNAL_REPAIR_DISCLOSURE_ITEM_ID @@ -164,13 +167,24 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal readOnly: host.readOnly, highestFence: () => host.state().highestFence, nextSequence: () => host.state().lastSequence + 1, - apply: (row) => applyJournalRow(host.state(), row), + apply: (row) => { + undo = beginJournalFoldUndo(host.state()) + applyJournalRow(host.state(), row) + }, writeStatus: (db, row) => writeStatus(db, host.state(), corruptAfter(row)), committed: (row) => { + undo?.commit() + undo = null host.setLoadCorrupt(corruptAfter(row)) host.notifyCommitted() }, - recoverFold: () => recoverJournalFold(host), + recoverFold: () => { + // What the projection read of the failed row must not answer for the next one at its seq. + statusProjection.invalidate() + const failed = undo + undo = null + recoverJournalFold(host, failed) + }, // Every rejection is a dispatch row through this one writer; the draft // returned-transition rides it so no path can bypass the hook. inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), @@ -212,12 +226,23 @@ function backfillSessionStatus( } /** - * An append whose transaction failed after its row was folded: the fold is ahead of the disk, and - * an in-place apply cannot be undone, so the chat is folded again from what committed. The host - * database rolls a stranded transaction back before it hands out the connection, so the re-read - * never sees the row that failed. A re-read that fails leaves the fold marked stale. + * An append whose transaction failed after its row was folded: the undo puts back what the row + * changed. If that fails, the chat is folded again from what committed (the host database rolls a + * stranded transaction back before it hands out the connection, so the re-read never sees the + * failed row), and if that fails too the fold is marked stale and re-read before its next use. */ -function recoverJournalFold(host: JournalStoreHost): void { +function recoverJournalFold(host: JournalStoreHost, undo: JournalFoldUndo | null): void { + try { + if (undo) { + undo.rollback() + return + } + } catch (error) { + console.warn('[agent-session-journal] undoing a failed append failed', { + sessionId: host.identity.sessionId, + error + }) + } try { const reloaded = replayJournal(host.database().db, host.identity.sessionId) if (reloaded) { From 7bb2d40aea8a651d2b0edb248f23ef0a57020dfe Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:53:39 -0700 Subject: [PATCH 030/107] refactor(native-chat): a failed undo marks the fold stale instead of re-reading it twice --- .../journal-session-state.test.ts | 23 +------------------ .../journal-store-collaborators.ts | 22 ++++-------------- .../agent-session-journal/journal-store.ts | 3 --- 3 files changed, 5 insertions(+), 43 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 579cd24a34c9..71aa8053930e 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -295,7 +295,7 @@ describe('a failed write leaves the fold equal to the disk (T1)', () => { expect(journal.submission('send-1')).toMatchObject({ dispatchState: 'pending' }) }) - it('re-reads the chat when the undo fails, and keeps the chat open', async () => { + it('re-reads before next use a fold whose undo failed, and keeps the chat open', async () => { const journal = await write('settled') const tip = journal.cursor() vi.spyOn(console, 'warn').mockImplementation(() => undefined) @@ -309,27 +309,6 @@ describe('a failed write leaves the fold equal to the disk (T1)', () => { await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') - expect(journal.cursor()).toEqual(tip) - expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) - }) - - it('re-reads before next use a fold whose undo and re-read both failed, and keeps the chat open', async () => { - const journal = await write('settled') - const tip = journal.cursor() - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - vi.mocked(JournalFoldUndo.beginJournalFoldUndo).mockImplementationOnce(() => ({ - commit: () => undefined, - rollback: () => { - throw new Error('undo failed') - } - })) - vi.mocked(JournalOpen.replayJournal).mockImplementationOnce(() => { - throw new Error('disk I/O error') - }) - failNextCommit() - - await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') - // The next read re-reads the disk instead of serving the fold that held the lost row. expect(journal.cursor()).toEqual(tip) expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, 'settled')!.state)) diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index b74259aab3b7..18e0d18401ff 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -20,7 +20,7 @@ import type { JournalRow } from './journal-row-schema' import type { AgentSessionJournal } from './journal-store' import type Database from '../../sqlite/sync-database' import { applyJournalRow } from './journal-reducer' -import { readJournalRowsAfterCursor, replayJournal } from './journal-open' +import { readJournalRowsAfterCursor } from './journal-open' import { readJournalSince } from './journal-cursor' import type { JournalReadSince } from './journal-store-contracts' import { JOURNAL_REPAIR_DISCLOSURE_ITEM_ID } from './journal-repair-disclosure' @@ -53,8 +53,6 @@ export type JournalStoreHost = { setReadOnly: (readOnly: boolean) => void cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void - /** The fold re-read from disk after a failed append. */ - replaceState: (state: JournalReducerState) => void /** That re-read failed too: the fold is re-read before its next use, never served as it is. */ markFoldStale: () => void /** Whether a fresh replay would report the history corrupt. */ @@ -227,9 +225,9 @@ function backfillSessionStatus( /** * An append whose transaction failed after its row was folded: the undo puts back what the row - * changed. If that fails, the chat is folded again from what committed (the host database rolls a - * stranded transaction back before it hands out the connection, so the re-read never sees the - * failed row), and if that fails too the fold is marked stale and re-read before its next use. + * changed. If it cannot, the fold is marked stale and folded again from what committed before its + * next use (the host database rolls a stranded transaction back before it hands out the connection, + * so that re-read never sees the failed row). */ function recoverJournalFold(host: JournalStoreHost, undo: JournalFoldUndo | null): void { try { @@ -243,17 +241,5 @@ function recoverJournalFold(host: JournalStoreHost, undo: JournalFoldUndo | null error }) } - try { - const reloaded = replayJournal(host.database().db, host.identity.sessionId) - if (reloaded) { - host.replaceState(reloaded.state) - return - } - } catch (error) { - console.warn('[agent-session-journal] re-reading a chat after a failed append failed', { - sessionId: host.identity.sessionId, - error - }) - } host.markFoldStale() } diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 5064a1e87a65..ecfd03de2adf 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -127,9 +127,6 @@ export class AgentSessionJournal { this.adoptLoadedJournal(loaded) this.onCommitted?.() }, - replaceState: (state) => { - this.state = state - }, markFoldStale: () => { this.foldStale = true }, From 7da72ff410b19df5e673843cab0621b9b3d10308 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:13:52 -0700 Subject: [PATCH 031/107] perf(native-chat): judge each item once for a chat's status; status write fixes from review - Each item's settlement facts and turn record are kept per item object (an entry is replaced on change, never edited), and the active turn comes from the same pass instead of a second scan. - The per-chat file copy drops the status of the rows it replaced instead of replaying the whole chat inside its transaction; the open after it writes the status from its own replay. - The epoch writes decide before their transaction that their new fold is not the store's own, and a stale fold refuses to be re-read inside a transaction, so a failed roll can never adopt the epoch it did not commit. - The persisted column is `lifecycle`, so it is not mistaken for the status a session list shows. - The status corpus gains seven chat states and the real fence; observers are shown to hear of a write only after COMMIT. --- .../journal-open-settlement-plan.ts | 77 +++++-- .../journal-per-session-import.test.ts | 8 +- .../journal-per-session-import.ts | 6 +- .../journal-session-state-test-corpus.ts | 200 +++++++++++------- .../journal-session-state-test-writes.ts | 95 +++++++++ .../journal-session-state.test.ts | 89 +++++++- .../journal-session-state.ts | 37 ++-- .../journal-store-collaborators.ts | 30 ++- .../agent-session-journal/journal-store.ts | 7 +- ...ured-agent-session-open-settlement.test.ts | 6 +- ...ctured-agent-session-startup-state.test.ts | 4 +- 11 files changed, 420 insertions(+), 139 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts diff --git a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts index 058e64316bea..58558c63b339 100644 --- a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts +++ b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts @@ -10,7 +10,8 @@ import type { AgentJournalItemBody, AgentJournalItemIdentity, AgentJournalRenderItem, - AgentJournalSubmission + AgentJournalSubmission, + AgentJournalTurnLifecycle } from '../../../shared/agent-session-journal-types' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' @@ -53,7 +54,8 @@ export function owesRecoveredDispatch(submission: AgentJournalSubmission): boole ) } -/** What a chat's settle would have to revise, counted from the same rules the settle applies. */ +/** What a chat's settle would have to revise, counted from the same rules the settle applies, and + * the turn the chat is running. */ export type JournalSettlementFacts = { /** A turn record or tool call still `running`, under a key a settle can revise. */ runningWork: boolean @@ -66,36 +68,75 @@ export type JournalSettlementFacts = { /** A working subagent roster or live background-task row; not counted on a corrupt load, whose * settle leaves rosters for the rebuild. */ liveChildWork: boolean + /** The newest turn record by sequence, when it is running (`activeStructuredAgentSessionTurnIdBySequence`). */ + activeTurnId: string | null +} + +type ItemFacts = { + running: boolean + prompt: boolean + roster: boolean + turn: AgentJournalTurnLifecycle | null +} + +// Per item object: a write replaces the entry it changes and never edits one, so an object's facts +// never change, and each append judges only the items it touched. +const ITEM_FACTS = new WeakMap() + +function itemFacts(item: AgentJournalRenderItem): ItemFacts { + let facts = ITEM_FACTS.get(item) + if (!facts) { + const running = isRunningJournalTurn(item) || isRunningToolCall(item) + const prompt = !running && openSettlementTerminalBody(item) !== null + const settlable = (running || prompt) && openSettlementItemIdentity(item) !== null + facts = { + running: settlable && running, + prompt: settlable && prompt, + roster: staleSubagentRosterRevision(item) !== null, + turn: readAgentJournalTurn(item.body) + } + ITEM_FACTS.set(item, facts) + } + return facts } export function journalSettlementFacts( fold: Pick, options: { settlesRosters: boolean } ): JournalSettlementFacts { - const facts: JournalSettlementFacts = { - runningWork: false, - pendingPrompts: false, - handedOverSends: 0, - queuedSends: 0, - liveChildWork: false - } + let handedOverSends = 0 + let queuedSends = 0 for (const submission of fold.submissions.values()) { if (isQueuedAgentJournalSubmission(submission)) { - facts.queuedSends += 1 + queuedSends += 1 } else if (owesRecoveredDispatch(submission)) { - facts.handedOverSends += 1 + handedOverSends += 1 } } + let runningWork = false + let pendingPrompts = false + let liveChildWork = false + let newestSequence = 0 + let newest: AgentJournalTurnLifecycle | null = null for (const item of fold.items.values()) { - facts.liveChildWork ||= options.settlesRosters && staleSubagentRosterRevision(item) !== null - const running = isRunningJournalTurn(item) || isRunningToolCall(item) - const prompt = !running && openSettlementTerminalBody(item) !== null - if ((running || prompt) && openSettlementItemIdentity(item)) { - facts.runningWork ||= running - facts.pendingPrompts ||= prompt + const facts = itemFacts(item) + runningWork ||= facts.running + pendingPrompts ||= facts.prompt + liveChildWork ||= options.settlesRosters && facts.roster + // Ties go to the later item, as the by-sequence reader decides them. + if (facts.turn && item.sequence >= newestSequence) { + newestSequence = item.sequence + newest = facts.turn } } - return facts + return { + runningWork, + pendingPrompts, + handedOverSends, + queuedSends, + liveChildWork, + activeTurnId: newest?.state === 'running' ? newest.turnId : null + } } function isRunningToolCall(item: Pick): boolean { diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts index f2a4cd21eb82..f7d1f5e63bdf 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts @@ -212,12 +212,12 @@ describe('importing a per-chat journal', () => { // T10: the copy's publish writes the chat's status for the rows it publishes, in the same // transaction, replacing whatever described the rows it replaced. - it('writes the chat status in the copy publish, for the rows it published', async () => { + it('drops the status of the rows the copy replaced, and the open after it writes the status again', async () => { const { epoch, rows } = await historyRows() await writeLegacyJournal(epoch, rows) const { db } = openTestJournalHostDatabase(root) db.prepare( - `INSERT INTO journal_session_state (session_id, status, active_turn_id, handed_over_sends, + `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at) VALUES (?, 'running', NULL, 3, 0, 0, '{"status":"working","latestPrompt":"stale"}', 1)` ).run(IDENTITY.sessionId) @@ -227,6 +227,10 @@ describe('importing a per-chat journal', () => { identity: IDENTITY, legacyDirectory: legacyDir() }) + expect(readTestJournalSessionStatus(root, IDENTITY.sessionId)).toBeNull() + + // As the chat's open does after its replay. + ;(await openChat()).backfillSessionStatus() const loaded = loadTestJournal(root, IDENTITY.sessionId)! expect(readTestJournalSessionStatus(root, IDENTITY.sessionId)).toEqual( diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts index 901d8add652e..76d0bd05c236 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.ts @@ -19,7 +19,7 @@ // an empty chat founded here would take a new epoch the next open's import could not reconcile. import { createHash } from 'node:crypto' -import { writeJournalSessionStatusFromDisk } from './journal-session-state' +import { deleteJournalSessionStatus } from './journal-session-state' import { existsSync } from 'node:fs' import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' @@ -225,7 +225,9 @@ async function copyLegacyJournal( ) } writePerSessionImportMarker(db, sessionId, legacy) - writeJournalSessionStatusFromDisk(db, sessionId) + // It described the rows this copy replaced. The replay that follows the copy writes it again + // (the chat's open), without a second whole-chat replay inside this transaction. + deleteJournalSessionStatus(db, sessionId) }) } diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts index c193d26ae50e..a81a68c09c46 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts @@ -1,85 +1,25 @@ -// Chats in every state an open's settlement plan can find, written through a real journal store, -// for the tests of the stored state and the plan that must agree about them. +// Chats in every state a settlement plan can find, written through a real journal store, for the +// tests of the stored status and the plan that must agree about them. -import { - AGENT_JOURNAL_THREAD_SCOPE, - type AgentJournalItemBody, - type AgentJournalItemIdentity -} from '../../../shared/agent-session-journal-types' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' +import { structuredAgentSessionPayloadFingerprint } from '../../../shared/structured-agent-session-mutation' import type { AgentSessionJournal } from './journal-store' +import { + assistantMessage, + codexItem, + CORPUS_FENCE, + item, + roster, + runningTool, + send, + settledTurn, + userMessage +} from './journal-session-state-test-writes' -/** The fence every case writes its content under; an `unverifiable` turn's writer. */ -export const CORPUS_FENCE = 3 - -const THREAD = 'thread-corpus' - -function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { - return { provider: 'codex', threadId: THREAD, turnId, ordinal } -} - -async function item( - journal: AgentSessionJournal, - identity: AgentJournalItemIdentity, - body: AgentJournalItemBody -): Promise { - await journal.appendItem(identity, body, { - fence: CORPUS_FENCE, - turnScope: AGENT_JOURNAL_THREAD_SCOPE - }) -} - -async function send( - journal: AgentSessionJournal, - clientMessageId: string, - text: string, - handoverRecorded?: true -): Promise { - await journal.appendSubmission({ - clientMessageId, - payloadFingerprint: `fp-${clientMessageId}`, - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, - fence: CORPUS_FENCE, - ...(handoverRecorded ? { handoverRecorded } : {}) - }) -} - -async function settledTurn(journal: AgentSessionJournal, turnId: string): Promise { - await send(journal, `send-${turnId}`, `asked ${turnId}`) - await journal.resolveDispatch({ - clientMessageId: `send-${turnId}`, - state: 'accepted', - providerIdentity: codexItem(turnId, 1), - fence: CORPUS_FENCE - }) - await item(journal, codexItem(turnId, 0), { - kind: 'turn', - turnId, - state: 'completed', - outcome: 'success', - startedAt: 10, - completedAt: 20 - }) - await item(journal, codexItem(turnId, 2), { - kind: 'message', - role: 'assistant', - blocks: [{ type: 'text', text: `answered ${turnId}` }] - }) -} - -function roster(state: 'working' | 'completed'): AgentJournalItemBody { - return { - kind: 'message', - role: 'system', - blocks: [ - { - type: 'subagent-group', - groupId: 'group-1', - agents: [{ id: 'child-1', label: 'reads', state, startedAt: 10 }] - } - ] - } -} +export { CORPUS_FENCE } from './journal-session-state-test-writes' /** Each case, written onto a freshly opened (empty) journal. */ export const JOURNAL_SESSION_STATE_CORPUS = { @@ -168,6 +108,101 @@ export const JOURNAL_SESSION_STATE_CORPUS = { 'settled roster': async (journal: AgentSessionJournal) => { await settledTurn(journal, 'turn-1') await item(journal, { provider: 'orca', clientMessageId: 'roster-1' }, roster('completed')) + }, + 'provider echo claiming a send': async (journal: AgentSessionJournal) => { + const body = userMessage('hello') + await journal.appendSubmission({ + clientMessageId: 'send-echo', + payloadFingerprint: structuredAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: journal.snapshot().sessionId, + fields: { body } + }), + body, + fence: CORPUS_FENCE + }) + await item(journal, codexItem('turn-2', 0), { + kind: 'turn', + turnId: 'turn-2', + state: 'running', + startedAt: 10 + }) + // The provider's copy of the user's own message, matched to the send by its fingerprint. + await item(journal, codexItem('turn-2', 1), userMessage('hello')) + await item(journal, codexItem('turn-2', 2), assistantMessage('working on it')) + }, + 'newest reply tombstoned': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await item(journal, codexItem('turn-1', 3), assistantMessage('a later reply')) + await journal.appendTombstone(codexItem('turn-1', 3), { fence: CORPUS_FENCE }) + }, + 'queued send handed over': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-queued', 'waiting its turn', true) + await journal.resolveDispatch({ + clientMessageId: 'send-queued', + state: 'pending', + turnScope: AGENT_JOURNAL_THREAD_SCOPE, + fence: CORPUS_FENCE + }) + }, + 'refused send': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-refused', 'nope') + await journal.resolveDispatch({ + clientMessageId: 'send-refused', + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('hostRestarted'), { + surface: 'rejection' + }), + fence: CORPUS_FENCE + }) + }, + 'running turn on a legacy status row': async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-9', 0), { + kind: 'status', + text: 'Codex is working', + turnLifecycle: { turnId: 'turn-9', state: 'running', startedAt: 10 } + }) + }, + 'running work settled by a batch': async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-1', 0), { + kind: 'turn', + turnId: 'turn-1', + state: 'running', + startedAt: 10 + }) + await item(journal, codexItem('turn-1', 4), runningTool) + await journal.appendLifecycleBatch({ + settlementId: 'settle-1', + fence: CORPUS_FENCE, + recovered: true, + mutations: [ + { + kind: 'item', + identity: codexItem('turn-1', 4), + body: { ...runningTool, state: 'failed' }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }, + { + kind: 'item', + identity: codexItem('turn-1', 0), + body: { + kind: 'turn', + turnId: 'turn-1', + state: 'interrupted', + startedAt: 10, + completedAt: 20 + }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } + ] + }) + }, + 'unknown send recovered': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-recovered', 'lost in a crash') + await journal.markPendingSubmissionsUnknown(CORPUS_FENCE) } } satisfies Record Promise> @@ -187,7 +222,14 @@ export const CORPUS_UNSETTLED: Record = { 'queued leftover': true, // Its verdict was decided when it settled; startup never revises it. 'unverifiable turn': false, - 'settled roster': false + 'settled roster': false, + 'provider echo claiming a send': true, + 'newest reply tombstoned': false, + 'queued send handed over': true, + 'refused send': false, + 'running turn on a legacy status row': true, + 'running work settled by a batch': false, + 'unknown send recovered': false } export const JOURNAL_SESSION_STATE_CASES = Object.keys(JOURNAL_SESSION_STATE_CORPUS).filter( diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts new file mode 100644 index 000000000000..21968f1b71db --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts @@ -0,0 +1,95 @@ +// How the chat-status corpus writes each chat, through a real journal store. + +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from './journal-store' + +/** The fence every case writes its content under; an `unverifiable` turn's writer. */ +export const CORPUS_FENCE = 3 + +const THREAD = 'thread-corpus' + +export function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: THREAD, turnId, ordinal } +} + +export async function item( + journal: AgentSessionJournal, + identity: AgentJournalItemIdentity, + body: AgentJournalItemBody +): Promise { + await journal.appendItem(identity, body, { + fence: CORPUS_FENCE, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) +} + +export async function send( + journal: AgentSessionJournal, + clientMessageId: string, + text: string, + handoverRecorded?: true +): Promise { + await journal.appendSubmission({ + clientMessageId, + payloadFingerprint: `fp-${clientMessageId}`, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + fence: CORPUS_FENCE, + ...(handoverRecorded ? { handoverRecorded } : {}) + }) +} + +export async function settledTurn(journal: AgentSessionJournal, turnId: string): Promise { + await send(journal, `send-${turnId}`, `asked ${turnId}`) + await journal.resolveDispatch({ + clientMessageId: `send-${turnId}`, + state: 'accepted', + providerIdentity: codexItem(turnId, 1), + fence: CORPUS_FENCE + }) + await item(journal, codexItem(turnId, 0), { + kind: 'turn', + turnId, + state: 'completed', + outcome: 'success', + startedAt: 10, + completedAt: 20 + }) + await item(journal, codexItem(turnId, 2), { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: `answered ${turnId}` }] + }) +} + +export function userMessage(text: string): AgentJournalItemBody { + return { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] } +} + +export function assistantMessage(text: string): AgentJournalItemBody { + return { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text }] } +} + +export const runningTool: Extract = { + kind: 'tool-call', + name: 'shell', + input: { command: 'ls' }, + state: 'running' +} + +export function roster(state: 'working' | 'completed'): AgentJournalItemBody { + return { + kind: 'message', + role: 'system', + blocks: [ + { + type: 'subagent-group', + groupId: 'group-1', + agents: [{ id: 'child-1', label: 'reads', state, startedAt: 10 }] + } + ] + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 71aa8053930e..616d3ef1491b 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -7,8 +7,11 @@ import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import { activeStructuredAgentSessionTurnIdBySequence } from '../../../shared/structured-agent-session-live-turn' import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' import { journalPragmaNumber } from './journal-database' +import { journalDatabasePath } from './journal-host-database' +import Database from '../../sqlite/sync-database' import { createTrackedJournalOpener, insertTestJournalRowJson, @@ -28,6 +31,7 @@ import { type JournalSessionStatus } from './journal-session-state' import { + CORPUS_FENCE, CORPUS_UNSETTLED, JOURNAL_SESSION_STATE_CASES, JOURNAL_SESSION_STATE_CORPUS, @@ -63,7 +67,9 @@ function open(sessionId: string): Promise { identity: identity(sessionId), stateDirectory: root, now: () => (clock += 1), - mintEpoch: () => `epoch-${sessionId}-${clock}` + mintEpoch: () => `epoch-${sessionId}-${clock}`, + // As production passes the record's fence. + currentFence: () => CORPUS_FENCE }) } @@ -77,7 +83,10 @@ function freshDerivation(sessionId: string): JournalSessionStatus { if (!loaded) { throw new Error(`no journal for ${sessionId}`) } - return deriveJournalSessionStatus(loaded.state, { settlesRosters: !loaded.corrupt }) + return deriveJournalSessionStatus(loaded.state, { + settlesRosters: !loaded.corrupt, + currentFence: CORPUS_FENCE + }) } const stored = (sessionId: string) => readTestJournalSessionStatus(root, sessionId) @@ -152,6 +161,39 @@ describe('the stored status follows every write (T3)', () => { expect(stored(name)).toEqual(freshDerivation(name)) expect(isUnsettledJournalSessionStatus(stored(name)!)).toBe(CORPUS_UNSETTLED[name]) expect(readUnsettledJournalSessionIds(db()).includes(name)).toBe(CORPUS_UNSETTLED[name]) + // The active turn the facts pass finds is the by-sequence reader's. + expect(stored(name)?.activeTurnId).toBe( + activeStructuredAgentSessionTurnIdBySequence( + loadTestJournal(root, name)!.state.items.values() + ) + ) + }) +}) + +describe('observers hear of a write only once it is committed (T13)', () => { + it('tells observers after COMMIT, when another connection already reads the row and its status', async () => { + const journal = await open('committed') + const seen: { inTransaction: boolean; rows: unknown; status: unknown }[] = [] + journal.observeCommits(() => { + const other = new Database(journalDatabasePath(root), { readonly: true }) + try { + seen.push({ + inTransaction: db().isTransaction, + rows: other + .prepare('SELECT COUNT(*) AS n FROM journal_rows WHERE session_id = ?') + .get('committed'), + status: other + .prepare('SELECT lifecycle FROM journal_session_state WHERE session_id = ?') + .get('committed') + }) + } finally { + other.close() + } + }) + + await note(journal, 'note-1') + + expect(seen).toEqual([{ inTransaction: false, rows: { n: 2 }, status: { lifecycle: 'idle' } }]) }) }) @@ -209,7 +251,7 @@ describe('a repair writes the status of what it leaves (T10)', () => { const journal = await write('running tool') const tip = journal.cursor() await journal.close() - expect(stored('running tool')).toMatchObject({ status: 'running' }) + expect(stored('running tool')).toMatchObject({ lifecycle: 'running' }) deleteJournalRepairedSuffix({ database: openTestJournalHostDatabase(root), @@ -222,7 +264,7 @@ describe('a repair writes the status of what it leaves (T10)', () => { }) expect(stored('running tool')).toEqual(freshDerivation('running tool')) - expect(stored('running tool')).toMatchObject({ status: 'idle' }) + expect(stored('running tool')).toMatchObject({ lifecycle: 'idle' }) }) }) @@ -318,16 +360,49 @@ describe('a failed write leaves the fold equal to the disk (T1)', () => { }) }) +describe('a stale fold is never read back inside a transaction (R2W-3)', () => { + it('keeps a stale fold on the committed epoch through a roll whose COMMIT fails', async () => { + const journal = await write('settled') + const committedEpoch = journal.epoch + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + // An append fails and its undo fails too: the fold is stale. + vi.mocked(JournalFoldUndo.beginJournalFoldUndo).mockImplementationOnce(() => ({ + commit: () => undefined, + rollback: () => { + throw new Error('undo failed') + } + })) + failNextCommit() + await expect(note(journal, 'lost')).rejects.toThrow('COMMIT failed') + // The next write is a roll, and its COMMIT fails too. + vi.restoreAllMocks() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + failNextCommit() + + await expect(journal.rollEpoch('handle_forked', 3)).rejects.toThrow('COMMIT failed') + + expect(journal.epoch).toBe(committedEpoch) + expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(committedEpoch) + // An acknowledged append lands where every replay reads it. + await note(journal, 'after-roll') + expect( + [...loadTestJournal(root, 'settled')!.state.items.keys()].some((id) => + id.includes('after-roll') + ) + ).toBe(true) + }) +}) + describe('epoch writes carry the status (T10)', () => { it('writes it for a new epoch, a roll and a replacement, in their transactions', async () => { const journal = await open('epochs') - expect(stored('epochs')).toMatchObject({ status: 'idle', summary: { status: null } }) + expect(stored('epochs')).toMatchObject({ lifecycle: 'idle', summary: { status: null } }) await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) - expect(stored('epochs')).toMatchObject({ status: 'running' }) + expect(stored('epochs')).toMatchObject({ lifecycle: 'running' }) await journal.rollEpoch('unreconcilable_prefix', 3) expect(stored('epochs')).toEqual(freshDerivation('epochs')) - expect(stored('epochs')).toMatchObject({ status: 'idle' }) + expect(stored('epochs')).toMatchObject({ lifecycle: 'idle' }) await journal.replaceEpochItems('legacy_import', 3, [ { diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index fd6b757e5874..1f96deca2a4c 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -12,7 +12,6 @@ import type Database from '../../sqlite/sync-database' import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' -import { activeStructuredAgentSessionTurnIdBySequence } from '../../../shared/structured-agent-session-live-turn' import { projectStructuredAgentSessionStatusState, type StructuredAgentSessionStatusProjection @@ -23,7 +22,7 @@ import { renderJournalState, type JournalReducerState } from './journal-reducer' /** Work a gone process can have left: running work, a waiting prompt, unanswered or queued sends, * or live child work. Startup settles exactly these chats. */ -const UNSETTLED = `status <> 'idle' OR handed_over_sends > 0 OR queued_sends > 0 OR live_child_work = 1` +const UNSETTLED = `lifecycle <> 'idle' OR handed_over_sends > 0 OR queued_sends > 0 OR live_child_work = 1` /** Recreated, empty, by every migration up to version 5: any row it held may predate writes an * older build made, and an open writes a missing row back. */ @@ -32,7 +31,7 @@ export function createJournalSessionStatusTable(db: Database.Database): void { DROP TABLE IF EXISTS journal_session_state; CREATE TABLE journal_session_state ( session_id TEXT PRIMARY KEY, - status TEXT NOT NULL, + lifecycle TEXT NOT NULL, active_turn_id TEXT, handed_over_sends INTEGER NOT NULL, queued_sends INTEGER NOT NULL, @@ -46,8 +45,10 @@ CREATE INDEX journal_session_state_unsettled ON journal_session_state (session_i } export type JournalSessionStatus = { - /** `running` covers a starting turn too: the journal records no separate start. */ - status: 'idle' | 'running' | 'attention' + /** What a settle would revise: work still running (a starting turn included: the journal records + * no separate start) or a prompt waiting. Not the status a session list shows, which is + * `summary.status`: that follows the newest turn and counts unanswered sends as working. */ + lifecycle: 'idle' | 'running' | 'attention' activeTurnId: string | null handedOverSends: number queuedSends: number @@ -70,8 +71,8 @@ export function deriveJournalSessionStatus( ): JournalSessionStatus { const facts = journalSettlementFacts(state, input) return { - status: facts.runningWork ? 'running' : facts.pendingPrompts ? 'attention' : 'idle', - activeTurnId: activeStructuredAgentSessionTurnIdBySequence(state.items.values()), + lifecycle: facts.runningWork ? 'running' : facts.pendingPrompts ? 'attention' : 'idle', + activeTurnId: facts.activeTurnId, handedOverSends: facts.handedOverSends, queuedSends: facts.queuedSends, liveChildWork: facts.liveChildWork, @@ -93,22 +94,22 @@ function projectSummary( export function isUnsettledJournalSessionStatus(status: JournalSessionStatus): boolean { return ( - status.status !== 'idle' || + status.lifecycle !== 'idle' || status.handedOverSends > 0 || status.queuedSends > 0 || status.liveChildWork ) } -const UPSERT_STATUS = `INSERT INTO journal_session_state (session_id, status, active_turn_id, +const UPSERT_STATUS = `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET - status = excluded.status, active_turn_id = excluded.active_turn_id, + lifecycle = excluded.lifecycle, active_turn_id = excluded.active_turn_id, handed_over_sends = excluded.handed_over_sends, queued_sends = excluded.queued_sends, live_child_work = excluded.live_child_work, summary_json = excluded.summary_json, last_activity_at = excluded.last_activity_at` -const STATUS_COLUMNS = `st.status AS status, st.active_turn_id AS active_turn_id, +const STATUS_COLUMNS = `st.lifecycle AS lifecycle, st.active_turn_id AS active_turn_id, st.handed_over_sends AS handed_over_sends, st.queued_sends AS queued_sends, st.live_child_work AS live_child_work, st.summary_json AS summary_json, st.last_activity_at AS last_activity_at` @@ -121,7 +122,7 @@ export function writeJournalSessionStatus( ): void { db.prepare(UPSERT_STATUS).run( sessionId, - status.status, + status.lifecycle, status.activeTurnId, status.handedOverSends, status.queuedSends, @@ -144,6 +145,10 @@ export function writeJournalSessionStatusFromDisk(db: Database.Database, session } } +export function deleteJournalSessionStatus(db: Database.Database, sessionId: string): void { + db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) +} + /** Whether the chat has a status row: an open writes one for a chat an older build last wrote. */ export function hasJournalSessionStatus(db: Database.Database, sessionId: string): boolean { return ( @@ -193,13 +198,13 @@ export function readUnsettledJournalSessionIds(db: Database.Database): string[] .flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) } -const STATUSES = ['idle', 'running', 'attention'] as const +const LIFECYCLES = ['idle', 'running', 'attention'] as const function parseStatusRow(row: Record): JournalSessionStatus | null { - const status = STATUSES.find((known) => known === row.status) + const lifecycle = LIFECYCLES.find((known) => known === row.lifecycle) const summary = parseSummary(row.summary_json) if ( - !status || + !lifecycle || !summary || typeof row.handed_over_sends !== 'number' || typeof row.queued_sends !== 'number' || @@ -208,7 +213,7 @@ function parseStatusRow(row: Record): JournalSessionStatus | nu return null } return { - status, + lifecycle, activeTurnId: typeof row.active_turn_id === 'string' ? row.active_turn_id : null, handedOverSends: row.handed_over_sends, queuedSends: row.queued_sends, diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 18e0d18401ff..487571107270 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -85,17 +85,22 @@ export type JournalStoreCollaborators = { export function createJournalStoreCollaborators(host: JournalStoreHost): JournalStoreCollaborators { const statusProjection = new JournalStatusProjection(host.state) - const writeStatus = (db: Database.Database, state: JournalReducerState, corrupt: boolean) => + /** `live`: the store's own fold, whose projection the status feed shares; an epoch's new fold + * projects its own. Decided by the caller, so nothing reads the store's fold inside the + * transaction. */ + const writeStatus = ( + db: Database.Database, + state: JournalReducerState, + corrupt: boolean, + live: boolean + ) => writeJournalSessionStatus( db, host.identity.sessionId, deriveJournalSessionStatus(state, { settlesRosters: !corrupt, currentFence: host.currentFence(), - // The live fold's projection, shared with the status feed; an epoch's new fold has its own. - ...(state === host.state() - ? { statusSummary: () => statusProjection.at(host.currentFence()).summary } - : {}) + ...(live ? { statusSummary: () => statusProjection.at(host.currentFence()).summary } : {}) }) ) // The open append's undo: what its row changed in the fold, put back if its transaction fails. @@ -117,7 +122,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal host.setLoadCorrupt(loaded.corrupt) host.adopt(loaded) }, - writeState: writeStatus + writeState: (db, state, corrupt) => writeStatus(db, state, corrupt, false) }) const queuedMessages = new JournalQueuedMessages({ sessionId: host.identity.sessionId, @@ -169,7 +174,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal undo = beginJournalFoldUndo(host.state()) applyJournalRow(host.state(), row) }, - writeStatus: (db, row) => writeStatus(db, host.state(), corruptAfter(row)), + writeStatus: (db, row) => writeStatus(db, host.state(), corruptAfter(row), true), committed: (row) => { undo?.commit() undo = null @@ -203,16 +208,23 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal /** Bookkeeping: a failure leaves the chat without a row, which its next open writes again. */ function backfillSessionStatus( host: JournalStoreHost, - writeStatus: (db: Database.Database, state: JournalReducerState, corrupt: boolean) => void + writeStatus: ( + db: Database.Database, + state: JournalReducerState, + corrupt: boolean, + live: boolean + ) => void ): void { const database = host.database() if (host.readOnly() || database.readOnly || host.importPending()) { return } try { + // Read before the transaction: a stale fold is re-read from disk, never inside one. + const state = host.state() database.transaction((db) => { if (!hasJournalSessionStatus(db, host.identity.sessionId)) { - writeStatus(db, host.state(), host.loadCorrupt()) + writeStatus(db, state, host.loadCorrupt(), true) } }) } catch (error) { diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index ecfd03de2adf..bfade83ef51f 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -158,7 +158,12 @@ export class AgentSessionJournal { /** Re-read from disk first while stale, so no reader or writer gets a fold the disk never held. */ private get state(): JournalReducerState { if (this.foldStale) { - const reloaded = replayJournal(this.database.db, this.identity.sessionId) + const db = this.database.db + if (db.isTransaction) { + // A transaction would read its own uncommitted rows back as the fold. + throw new Error(`agent-session journal ${this.identity.sessionId} is stale mid-transaction`) + } + const reloaded = replayJournal(db, this.identity.sessionId) if (!reloaded) { throw new Error(`agent-session journal ${this.identity.sessionId} is gone`) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts index 4966388f6a59..68d7f9d14d4c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -204,21 +204,21 @@ describe('the stored status and the plan agree (T4)', () => { ) expect(openSettlementPlanIsEmpty(plan)).toBe(true) - expect(storedStatus('unkeyed')).toMatchObject({ status: 'idle' }) + expect(storedStatus('unkeyed')).toMatchObject({ lifecycle: 'idle' }) expect(selected('unkeyed')).toBe(false) }) it('never selects a chat to revise a verdict: an unverifiable turn stays as it settled (D16 gone)', async () => { const journal = await open('unverifiable') await JOURNAL_SESSION_STATE_CORPUS['unverifiable turn'](journal) - expect(storedStatus('unverifiable')).toMatchObject({ status: 'idle' }) + expect(storedStatus('unverifiable')).toMatchObject({ lifecycle: 'idle' }) expect(selected('unverifiable')).toBe(false) }) it('selects a chat whose only debt is a queued leftover, and settles it (T15b)', async () => { const journal = await open('queued') await JOURNAL_SESSION_STATE_CORPUS['queued leftover'](journal) - expect(storedStatus('queued')).toMatchObject({ status: 'idle', queuedSends: 1 }) + expect(storedStatus('queued')).toMatchObject({ lifecycle: 'idle', queuedSends: 1 }) const plan = planOpenSettlement(journal, null, { settlesRosters: true }) expect(plan.leftoverQueued).toEqual(['send-queued']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 8bb4f68169c2..f24ac4ba8e52 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -295,7 +295,7 @@ describe('startup opens only what it must (T6, T7, T14)', () => { for (const sessionId of ['session-crashed', ...unlisted]) { // The unanswered send is now recovered doubt, which projects as no running request. expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ - status: 'idle', + lifecycle: 'idle', handedOverSends: 0, summary: { status: null } }) @@ -393,7 +393,7 @@ describe('one awaited settle covers a chat whose tab closes meanwhile (R1T-4)', await rig.host.settleOwedSessions(listed) expect(readTestJournalSessionStatus(rig.root, 'session-closing')).toMatchObject({ - status: 'idle', + lifecycle: 'idle', handedOverSends: 0 }) expect(rig.host.hasSession('session-closing')).toBe(false) From 93cb6cc9cbb85f8b3ebfab526ac4e3e1b460a6e6 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:22:54 -0700 Subject: [PATCH 032/107] fix(native-chat): chat commands wait for the startup settle before any chat's lock The startup gate was awaited inside the command open, which every caller runs under the chat's lock; the settle needs that lock too, so a command that reached a crashed chat first held the settle until the ceiling. Commands now wait at their entry, before the lock: mutations, the option read, attach and `conversation()`. A send, approval or question answer or option change that arrives before the host is built builds it and is held, not refused. The ceiling drops to 10 s, under every client's request timeout. Desktop starts the step behind the first-window services only. Rows whose record is gone or whose provider this host does not serve are dropped instead of selected every boot. The step requires seed and settle, and its dead lease-failure rethrow and the test-only plan helper are gone. --- docs/reference/agent-status-store.md | 8 ++- .../journal-session-state.ts | 3 +- ...ured-agent-session-attach-orchestration.ts | 16 ++++- ...red-agent-session-conversation-lifetime.ts | 3 +- .../structured-agent-session-host-delivery.ts | 11 +--- ...structured-agent-session-host-mutations.ts | 35 ++++++----- ...ured-agent-session-open-settlement.test.ts | 11 +++- ...tructured-agent-session-open-settlement.ts | 16 ++--- .../structured-agent-session-options-read.ts | 2 + ...ctured-agent-session-startup-state.test.ts | 62 +++++++++++++++++++ .../structured-agent-session-startup-state.ts | 25 ++++++++ .../structured-agent-session-status-feed.ts | 6 +- ...tore-structured-agent-session-tabs-once.ts | 5 +- ...runtime-structured-session-restore.test.ts | 23 ++++++- ...d-agent-session-send-compatibility.test.ts | 18 +++++- ...ctured-agent-session-send-compatibility.ts | 9 ++- .../rpc/methods/structured-agent-session.ts | 8 ++- .../structured-agent-session-startup-gate.ts | 13 ++-- ...-agent-session-startup-restoration.test.ts | 7 +-- .../structured-agent-session-startup-step.ts | 29 +++------ ...ctured-agent-session-support-probe.test.ts | 2 + .../startup/main-process-runtime-launch.ts | 6 +- ...ed-session-reopened-tab-close.unit.test.ts | 2 + 23 files changed, 232 insertions(+), 88 deletions(-) diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index b0f7dc65415e..49ed1b35b547 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -131,9 +131,11 @@ Two rules the ingest must keep: - **Never persist a structured row in the store.** The chat journal database is the durable truth for a structured session. Each chat's status is stored there beside its journal (`journal_session_state`), written in the same - transaction as the journal rows it describes, so it is always current. At - host startup the host republishes a settled chat's row into the store without - opening the chat, and settles a chat a gone process left with work first. A structured row in `last-status.json` would + transaction as the journal rows it describes, so it is always current; a + chat last written before that table existed has no row until it is opened. + At host startup the host republishes a settled chat's row into the store + without opening the chat, and settles a chat a gone process left with work + first. A structured row in `last-status.json` would hydrate as `restoredUnconfirmed` and fight that republish, so the serializer still skips rows carrying `structuredHost`, and hydrate still drops any found on disk. Applying one therefore also skips diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index 1f96deca2a4c..c6fbc3a87ec5 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -8,7 +8,8 @@ // and seeds every other chat's status from them. // // The table arrives with schema version 5, so a build older than that opens the database read-only -// and never writes a journal row without its status. +// and never writes a journal row without its status. Rows carry no rules version: a change to what +// the derivation or the selection reads needs a migration that recreates the table. import type Database from '../../sqlite/sync-database' import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index 9cfdf6a9b87c..1e2e09621773 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -66,6 +66,18 @@ export function attachStructuredAgentSessionUnderSerialize( return context.tasks.trackAttach(runAttach(context, callerKey, params, options)) } +/** Startup's settle first, before the chat's lock, which the settle takes too; queued at once when + * startup is done, so attaches keep the order they arrived in. */ +function waitThenSerialize( + context: StructuredAgentSessionAttachContext, + sessionId: string, + task: () => Promise +): Promise { + const startup = context.deps.commandsReady?.() + const run = () => context.serialize(sessionId, task) + return startup ? startup.then(run) : run() +} + export function attachStructuredAgentSession( context: StructuredAgentSessionAttachContext, callerKey: string, @@ -76,7 +88,9 @@ export function attachStructuredAgentSession( // evicts, so no child is spawned behind the eviction and orphaned. const run = (recordPhase?: AgentSessionCreatePhaseRecorder) => context.tasks.trackAttach( - context.serialize(sessionId, () => runAttach(context, callerKey, params, { recordPhase })) + waitThenSerialize(context, sessionId, () => + runAttach(context, callerKey, params, { recordPhase }) + ) ) if (params.envelope.expectedRuntimeFence !== null) { return run() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 6c1c55f34a44..d3071e9a8013 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -103,7 +103,8 @@ export function createStructuredAgentSessionConversationLifetime(host: { * drop it after. */ conversation: async (sessionId: string): Promise => { - // Startup's settle first: no command acts on a chat a gone process left mid-turn. + // Startup's settle first, before this chat's lock, which the settle takes too. No caller + // holds a chat's lock here. const startup = deps().commandsReady?.() if (startup) { await startup diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index 95040b7ba206..cdb7b473ae23 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -138,15 +138,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { loop.dispose() stopResettling() }, - open: async (sessionId, options) => { - // Every command that opens a chat waits for startup's settle, as `conversation()` does. - await deps.commandsReady?.() - return openStructuredAgentSessionConversation( - { deps, sessions, adoptOpened }, - sessionId, - options - ) - } + open: (sessionId, options) => + openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index 3055bb2d8789..f0a52429d5d0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -88,21 +88,26 @@ export function mutateStructuredAgentSession( plan: MutationPlan, prepareSession?: AgentSessionMutationRequest['prepareSession'] ): Promise> { - return context.serialize(envelope.sessionId, () => - admitAndRunAgentSessionMutation({ - store: context.deps.store, - adapter: context.deps.adapter, - callerKey: caller.callerKey, - envelope, - plan, - journal: () => context.sessions.get(envelope.sessionId)?.journal, - prepareSession, - publish: (journal) => context.publish(envelope.sessionId, journal), - flushStreamedEvents: context.flushStreamedEvents, - providerChildPhase: () => context.sessions.get(envelope.sessionId)?.child?.phase, - now: () => context.now() - }) - ) + const run = () => + context.serialize(envelope.sessionId, () => + admitAndRunAgentSessionMutation({ + store: context.deps.store, + adapter: context.deps.adapter, + callerKey: caller.callerKey, + envelope, + plan, + journal: () => context.sessions.get(envelope.sessionId)?.journal, + prepareSession, + publish: (journal) => context.publish(envelope.sessionId, journal), + flushStreamedEvents: context.flushStreamedEvents, + providerChildPhase: () => context.sessions.get(envelope.sessionId)?.child?.phase, + now: () => context.now() + }) + ) + // Startup's settle first, before the chat's lock: the settle takes that lock too. Queued at once + // when startup is done, so commands keep the order they arrived in. + const startup = context.deps.commandsReady?.() + return startup ? startup.then(run) : run() } export function sendStructuredAgentSessionTurn( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts index 68d7f9d14d4c..d2067ec3f385 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -31,11 +31,20 @@ import { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { appendOpenSettlement, - openSettlementPlanIsEmpty, planOpenSettlement, + type OpenSettlementPlan, type OpenSettlementRecordFacts } from './structured-agent-session-open-settlement' +function openSettlementPlanIsEmpty(plan: OpenSettlementPlan): boolean { + return ( + plan.recoveredDispatches.length === 0 && + plan.leftoverQueued.length === 0 && + (plan.goneGeneration?.mutations.length ?? 0) === 0 && + plan.rosters.length === 0 + ) +} + const journals = createTrackedJournalOpener() let root: string let clock = 1_000 diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts index 1414f85ec616..f329254ceb4f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.ts @@ -1,8 +1,9 @@ // Everything a non-acquisition open writes because an earlier host process is gone, as one plan. // -// Built from the per-entity rules in journal-open-settlement-plan.ts, which the stored "owes work" -// flag reads too, so a chat's flag says it owes work exactly when this plan is non-empty. The open -// appends exactly this plan, and each entry revises its entity out of the state that selected it. +// Built from the per-entity rules in journal-open-settlement-plan.ts, which each chat's stored +// status counts with too, so startup selects a chat when this plan has work beyond revising an +// earlier verdict. The open appends exactly this plan, and each entry revises its entity out of +// the state that selected it. import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { @@ -102,15 +103,6 @@ export function planOpenSettlement( } } -export function openSettlementPlanIsEmpty(plan: OpenSettlementPlan): boolean { - return ( - plan.recoveredDispatches.length === 0 && - plan.leftoverQueued.length === 0 && - (plan.goneGeneration?.mutations.length ?? 0) === 0 && - plan.rosters.length === 0 - ) -} - /** * Settles whatever a generation with no child in this process left running: found when a new child * is acquired, or when a chat is reopened. Derived from the journal and the lease's death evidence diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts index 3bad8d9789f1..7b50ce346d6b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts @@ -96,6 +96,8 @@ export async function readStructuredAgentSessionOptions( sessionId: string ): Promise { const { adapter, store } = context.deps + // Startup's settle first, before the chat's lock, which the settle takes too. + await context.deps.commandsReady?.() const live = await context.serialize(sessionId, async () => { const session = await context.openConversation(sessionId).catch((error: unknown) => { throw journalOpenReadRefusal(error) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index f24ac4ba8e52..7b97e7a4c1f5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -16,6 +16,7 @@ import { import Database from '../../sqlite/sync-database' import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' +import { editPersistedTestAgentSessionStore } from '../../runtime/agent-session-record-store-test-harness' import { createRestTestRig, latestRestTestStatus, @@ -447,3 +448,64 @@ describe('the startup gate holds chat commands and never refuses them', () => { expect(gate.ready()).toBeNull() }) }) + +describe('commands held for the real startup settle never deadlock it (R2T-1)', () => { + const CEILING_MS = 3_000 + + it('lets a command issued before the settle, one during it and a healthy read through as the settle ends', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const rig = await newRig() + for (const sessionId of ['session-a', 'session-b']) { + await crashMidSend(rig, sessionId) + } + await restTestChat(rig, 'session-healthy', { message: 'fine' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + await rig.boot({ commandsReady: gate.ready }) + const listed = listedIds(rig) + await rig.host.reconcileRestartLeases() + rig.host.seedStoredStatuses(listed) + const started = Date.now() + const elapsed = () => Date.now() - started + + // Before the settle: a send to a crashed chat, and the option read a chat pane fires on mount. + const sendBefore = sendRestTestMessage(rig, 'session-a', 'during startup').then(elapsed) + const optionsBefore = rig.host.readOptions('session-a').then(elapsed, elapsed) + const settled = rig.host.settleOwedSessions(listed) + gate.openWhen(settled) + // During the settle: the second crashed chat, which the settle has not reached yet. + const optionsDuring = rig.host.readOptions('session-b').then(elapsed, elapsed) + const healthyRead = rig.host.journalSnapshot('session-healthy').then(elapsed) + const settleEnded = await settled.then(elapsed) + + const answered = await Promise.all([sendBefore, optionsBefore, optionsDuring, healthyRead]) + expect(settleEnded).toBeLessThan(CEILING_MS / 2) + for (const at of answered) { + expect(at).toBeLessThan(CEILING_MS / 2) + } + expect(gate.ready()).toBeNull() + }, 20_000) +}) + +describe('a stored status no settle here can clear (R2A-4)', () => { + it('drops the row of a chat whose record is gone or whose provider this host does not serve', async () => { + const rig = await newRig() + await crashMidSend(rig, 'session-gone', false) + await crashMidSend(rig, 'session-elsewhere', false) + await rig.crash() + await editPersistedTestAgentSessionStore(rig.root, (persisted) => { + delete persisted.records['session-gone'] + }) + rig.unsupportedWorkspaceIds.add(rig.store.getRecord('session-elsewhere')!.location.workspaceId) + + await rig.boot() + await startup(rig, listedIds(rig)) + + for (const sessionId of ['session-gone', 'session-elsewhere']) { + expect(readTestJournalSessionStatus(rig.root, sessionId)).toBeNull() + } + expect(opened(rig, ['session-gone', 'session-elsewhere'])).toEqual([]) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index eaa93db7b6f0..945e895eb0b7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -17,6 +17,7 @@ import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { + deleteJournalSessionStatus, isUnsettledJournalSessionStatus, readJournalSessionStatuses, readUnsettledJournalSessionIds @@ -136,6 +137,7 @@ async function settleOwedSessions( for (const sessionId of readUnsettledJournalSessionIds(database.db)) { const record = deps.openDeps.store.getRecord(sessionId) if (!record || !deps.supportsRecord(record)) { + dropUnreachableStatus(database, sessionId, record) continue } if (listedOrder.has(sessionId)) { @@ -171,6 +173,29 @@ async function settleOwedSessions( } } +/** + * A row no settle here can clear: its chat's record is gone, or this host does not serve its + * provider. Dropped, so it is not selected every boot; an open writes it again if the chat is ever + * opened here. Kept while the records import is owed, which may still bring the record. + */ +function dropUnreachableStatus( + database: StructuredAgentSessionStartupStateDeps['openDeps']['journalDatabase'], + sessionId: string, + record: AgentSessionRecord | null +): void { + if (!record && database.legacyRecordImportOwed) { + return + } + try { + deleteJournalSessionStatus(database.db, sessionId) + } catch (error) { + console.warn('[structured-agent-session] dropping an unreachable chat status failed', { + sessionId, + error + }) + } +} + /** Every lease a crash left `recovering`, listed or not: a provider process that outlived the crash * is stopped and its death recorded, so the settle below reads one verdict per turn. */ async function resolveRecoveringLeases(deps: StructuredAgentSessionStartupStateDeps) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index 91264592aec2..4fa3451607c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -7,9 +7,9 @@ // // The last projection is kept after the session's provider child is evicted: an idle session is // still idle without a process, and a renderer that reloads must not lose every settled row until -// each chat is reopened. At startup, a settled chat's row is seeded from the state stored beside its -// journal (journal-session-state.ts) without opening it; a chat that owes work is settled by its -// open, which publishes as any open does. +// each chat is reopened. At startup, a settled chat's row is seeded from the status stored beside +// its journal (journal-session-state.ts) without opening it; a chat a gone process left with work +// is settled by its open, which publishes as any open does. import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 4e14fe7a0dae..03bcf0654bd9 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -89,8 +89,9 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const session of restored) { this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } - // Startup already seeded every settled chat's row and is settling the ones that owe work; this - // opens only what stored state could not answer (a stale row, a per-chat file not yet copied). + // Startup already seeded every settled chat's row and is settling the ones a gone process left + // with work; this opens only what stored status could not answer (no row yet, a row this build + // cannot read, a per-chat file not yet copied). // Whoever answers with this list starts it, once that answer is out. const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds this.owedStructuredAgentSessionHistoryRestore = host diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 0e1fdbc9b76b..9e8ca9a83a7c 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -18,7 +18,11 @@ describe('structured session cold restoration', () => { internal.hasPersistedStructuredAgentSessionStore = () => false internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ reconcileRestartLeases } as never) + setStructuredAgentSessionHost({ + reconcileRestartLeases, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined + } as never) await runtime.prepareStructuredAgentSessionStartupRestoration() @@ -41,7 +45,12 @@ describe('structured session cold restoration', () => { internal.hasPersistedStructuredAgentSessionStore = () => true internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ reconcileRestartLeases, restoreReadableSessions } as never) + setStructuredAgentSessionHost({ + reconcileRestartLeases, + restoreReadableSessions, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined + } as never) await runtime.prepareStructuredAgentSessionStartupRestoration() @@ -75,6 +84,8 @@ describe('structured session cold restoration', () => { internal.ensureStructuredAgentSessionHost = ensureHost setStructuredAgentSessionHost({ reconcileRestartLeases, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined, restoreReadableSessions, listSessionTabs: () => [] } as never) @@ -142,6 +153,8 @@ describe('structured session cold restoration', () => { internal.ensureStructuredAgentSessionHost = async () => undefined setStructuredAgentSessionHost({ reconcileRestartLeases: async () => undefined, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined, getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: ['session-survives-rollback'] @@ -202,6 +215,8 @@ describe('structured session cold restoration', () => { internal.ensureStructuredAgentSessionHost = async () => undefined setStructuredAgentSessionHost({ reconcileRestartLeases: async () => undefined, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined, getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: [] }), restoreReadableSessions, listSessionTabs: () => [] @@ -236,6 +251,8 @@ describe('structured session cold restoration', () => { internal.ensureStructuredAgentSessionHost = async () => undefined setStructuredAgentSessionHost({ reconcileRestartLeases: async () => undefined, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined, restoreReadableSessions: async () => undefined, close: closeStructuredSession, setSessionTabVisibility, @@ -351,6 +368,8 @@ describe('structured session cold restoration', () => { internal.ensureStructuredAgentSessionHost = async () => undefined setStructuredAgentSessionHost({ reconcileRestartLeases: async () => undefined, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined, restoreReadableSessions: async () => undefined, listSessionTabs: () => [ { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts index 59bc914558e1..20de95a370d8 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts @@ -1,7 +1,7 @@ // Which clients get a send answered at acceptance, and which have their reply held until the // message is handed over: a client that cannot show a rejection after `pending` must not see one. -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY @@ -66,6 +66,22 @@ describe('agentSession.send reply timing', () => { }) }) +describe('a send that arrives before the host is built', () => { + it('builds the host and is answered by it, rather than refused', async () => { + clearStructuredHostStub() + hostCalls.send?.mockReset() + const ensureStructuredAgentSessionHost = vi.fn(async () => installStructuredHostStub()) + + const response = await call('agentSession.send', sendParams(), STRUCTURED_CLIENT, { + ensureStructuredAgentSessionHost + }) + + expect(ensureStructuredAgentSessionHost).toHaveBeenCalledOnce() + expect(response).not.toMatchObject({ ok: false }) + expect(hostCalls.send).toHaveBeenCalledOnce() + }) +}) + function pendingSendResult() { return { ok: true, diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts index 47afaa8ac735..0c65d65eb7e1 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts @@ -6,7 +6,10 @@ import { agentSessionSendSubmission } from '../../../../shared/agent-session-wir import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import type { RpcContext } from '../core' -import { requireStructuredHost, structuredCallerFor } from './structured-agent-session-gate' +import { + requireInstalledStructuredHost, + structuredCallerFor +} from './structured-agent-session-gate' /** * A send answers once the host accepts it. A client that predates that answer cannot show a @@ -19,7 +22,9 @@ export async function sendStructuredAgentSessionForClient( params: Parameters[1], context: RpcContext ) { - const host = requireStructuredHost(context) + // Built first when this process has none yet, so a send that arrives during startup is held + // for the startup settle instead of refused. + const host = await requireInstalledStructuredHost(context) // Only a client's own send lifts a Stop's queue pause; host-internal senders never do. const result = await host.send(structuredCallerFor(context), { ...params, userSend: true }) const capabilities = context.clientCapabilities ?? [] diff --git a/src/main/runtime/rpc/methods/structured-agent-session.ts b/src/main/runtime/rpc/methods/structured-agent-session.ts index b97da008a402..f0fa500e51cf 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.ts @@ -20,6 +20,8 @@ import { defineMethod, defineStreamingMethod, type RpcContext } from '../core' import { ensureStructuredHostInstalled as ensureHostInstalled, requireInstalledStructuredHost as requireInstalledHost, + // A chat command builds the host when it is this process's first: held for startup, not refused. + requireInstalledStructuredHost as commandHost, requireStructuredCapability, requireStructuredCleanupHost, requireStructuredHost as requireHost, @@ -230,18 +232,18 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ name: 'agentSession.respondToApproval', params: RespondParams, handler: async (params, ctx) => - requireHost(ctx).respondToPrompt(callerFor(ctx), { ...params, kind: 'approval' }) + (await commandHost(ctx)).respondToPrompt(callerFor(ctx), { ...params, kind: 'approval' }) }), defineMethod({ name: 'agentSession.respondToQuestion', params: RespondToQuestionParams, handler: async (params, ctx) => - requireHost(ctx).respondToPrompt(callerFor(ctx), { ...params, kind: 'question' }) + (await commandHost(ctx)).respondToPrompt(callerFor(ctx), { ...params, kind: 'question' }) }), defineMethod({ name: 'agentSession.setOption', params: SetOptionParams, - handler: async (params, ctx) => requireHost(ctx).setOption(callerFor(ctx), params) + handler: async (params, ctx) => (await commandHost(ctx)).setOption(callerFor(ctx), params) }), defineMethod({ name: 'agentSession.handoffStatus', diff --git a/src/main/runtime/structured-agent-session-startup-gate.ts b/src/main/runtime/structured-agent-session-startup-gate.ts index dee92bdff88d..6702f13bfa63 100644 --- a/src/main/runtime/structured-agent-session-startup-gate.ts +++ b/src/main/runtime/structured-agent-session-startup-gate.ts @@ -1,10 +1,13 @@ // Holds every structured chat command until host startup has settled the chats a gone process left -// with work, so no command acts on a chat whose crashed turn still reads as running. Commands wait; -// none is refused. It always opens: when the settle ends, however it ends, or at a ceiling, so a -// startup fault never strands a command. +// with work. Every open settles its own chat too, so what the hold adds is order: each crashed turn +// gets its verdict from the death evidence startup records first, never "Couldn't confirm" from an +// open that ran before it. Commands wait at their entry, before any chat's lock (the settle takes +// those locks); none is refused. It always opens: when the settle ends, however it ends, or at a +// ceiling, so a startup fault never strands a command. -// Far past any measured settle; only a hung step reaches it. -const STARTUP_GATE_CEILING_MS = 120_000 +// Under the shortest client request timeout (15 s): a held command lets go before its caller gives +// up, and an open settles its own chat, so letting go early is safe. +const STARTUP_GATE_CEILING_MS = 10_000 export class StructuredAgentSessionStartupGate { private pending: Promise | null = null diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index 1f6df6b54555..fa7d546c3ca4 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -1,6 +1,6 @@ // The host's own startup step, run from `prepare` whether or not any client ever lists a tab: a // headless host has its seeded statuses and its settled crashed chats, and no failure in the step -// (the lease check, one chat's open, one chat's settlement) costs startup or the other chats. +// (one chat's open, one chat's settlement) costs startup or the other chats. import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { @@ -128,8 +128,6 @@ it('never fails startup: each failure is logged by chat and the rest still settl await rig.crash() await rig.boot() const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const leaseCheck = new Error('lease check failed') - vi.spyOn(rig.host, 'reconcileRestartLeases').mockRejectedValueOnce(leaseCheck) rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { if (sessionId === 'session-open-fails') { throw new Error('EACCES: permission denied') @@ -147,8 +145,7 @@ it('never fails startup: each failure is logged by chat and the rest still settl ) const runtime = restartedRuntime() - // The lease check's failure is still the step's answer, unchanged. - await expect(runtime.prepareStructuredAgentSessionStartupRestoration()).rejects.toBe(leaseCheck) + await runtime.prepareStructuredAgentSessionStartupRestoration() expect(latestRestTestStatus(rig, 'session-settled')).toMatchObject({ status: 'idle' }) await vi.waitFor(() => expect(owes('session-fine')).toBe(false)) diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index 23050b8dd7e6..c00d2c8aef08 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -6,24 +6,21 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' -/** Named from the host so a rename fails to compile. Only the lease check is required: a host - * stand-in without stored state seeds and settles nothing, and lists the saved session's chats. */ +/** Named from the host so a rename fails to compile. */ export type StructuredAgentSessionStartupHost = Pick< StructuredAgentSessionHost, - 'reconcileRestartLeases' + 'reconcileRestartLeases' | 'seedStoredStatuses' | 'settleOwedSessions' > & - StructuredAgentSessionListingHost & - Partial> + StructuredAgentSessionListingHost type StructuredAgentSessionListingHost = Partial< Pick > /** - * Answers the listed chats stored state could not answer, which the post-listing restore opens. - * Death evidence the lease check writes must exist before the settle takes its verdicts; a failed - * check leaves them `unverifiable`, which is safe, so the seed and settle still run and the failure - * is thrown after them. + * Answers the listed chats stored status could not answer, which the post-listing restore opens. + * The lease check runs first: the death evidence it writes is what the settle's verdicts read. It + * reports its own failures and never throws. */ export async function runStructuredAgentSessionStartupStep( host: StructuredAgentSessionStartupHost, @@ -31,19 +28,11 @@ export async function runStructuredAgentSessionStartupStep( /** The settle, which chat commands wait for; it never rejects. */ onSettling: (settled: Promise) => void ): Promise { - let leaseFailure: { error: unknown } | null = null - try { - await host.reconcileRestartLeases() - } catch (error) { - leaseFailure = { error } - } + await host.reconcileRestartLeases() const listedIds = listedStructuredAgentSessionIds(host, savedSession) - const background = host.seedStoredStatuses?.(listedIds) ?? listedIds + const background = host.seedStoredStatuses(listedIds) // Not awaited here: the tab list and paint never wait on it; chat commands do. - onSettling(host.settleOwedSessions?.(listedIds) ?? Promise.resolve()) - if (leaseFailure) { - throw leaseFailure.error - } + onSettling(host.settleOwedSessions(listedIds)) return background } diff --git a/src/main/runtime/structured-agent-session-support-probe.test.ts b/src/main/runtime/structured-agent-session-support-probe.test.ts index ea0ab3298d9b..604d79c91997 100644 --- a/src/main/runtime/structured-agent-session-support-probe.test.ts +++ b/src/main/runtime/structured-agent-session-support-probe.test.ts @@ -35,6 +35,8 @@ function stubStructuredHostInstall(runtime: OrcaRuntimeService): { // still returns the right answer and fails on the install effects alone. const host = { reconcileRestartLeases: vi.fn(async () => {}), + seedStoredStatuses: vi.fn((ids: readonly string[]) => [...ids]), + settleOwedSessions: vi.fn(async () => undefined), supportsCreate: (location: { executionHostId: string; wslDistro: string | null }) => location.executionHostId === 'local' && location.wslDistro === null } diff --git a/src/main/startup/main-process-runtime-launch.ts b/src/main/startup/main-process-runtime-launch.ts index 4a8a6fe132bb..3af2e34c435a 100644 --- a/src/main/startup/main-process-runtime-launch.ts +++ b/src/main/startup/main-process-runtime-launch.ts @@ -340,8 +340,10 @@ export async function initializeMainProcessRuntimeLaunch( bindTerminalRuntimeStartupServices(Promise.resolve(startTerminalRuntimeStartupServices())) } // The host seeds its chats' statuses and settles crashed ones itself, so a window, a remote - // client or neither sees the same state. Behind the barriers the renderer's own call awaits. - void Promise.all([state.firstWindowStartupServicesReady, state.managedWslCliStartupBarrierReady]) + // client or neither sees the same state. Only behind the first-window services: the PTY + // inventory it takes before the lease check needs the daemon provider. The managed-WSL barrier + // gates terminal spawns, which this step makes none of. + void state.firstWindowStartupServicesReady .then(() => runtime.prepareStructuredAgentSessionStartupRestoration()) .catch((error: unknown) => console.warn('[structured-agent-session] startup failed', error)) app.on('activate', options.handleMacAppActivation) diff --git a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts index 35547145c32f..2a769b62fc62 100644 --- a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts +++ b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts @@ -128,6 +128,8 @@ async function setup() { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: restore and tab close call only these host members. { reconcileRestartLeases: async () => undefined, + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined, restoreReadableSessions: async () => undefined, close: async (sessionId: string) => { hostSessionCloses.push(sessionId) From 661984b443b775365b6198f71fe63a4b002053e8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:32:10 -0700 Subject: [PATCH 033/107] test(native-chat): read a quit's turn verdict the way the next launch does The chat history restore now opens nothing once the host has begun quitting, so the two close-verdict tests that quit and then restore the chat on the same host could no longer open it. They now read the settled turn through a fresh host over the same store and journal, which is what the next launch sees. --- ...ctured-agent-session-close-verdict.test.ts | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts index 83d80437de3c..ea0870f480e4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts @@ -124,14 +124,33 @@ async function runningTurn(): Promise { } /** What the settle wrote, read back from the journal the next reader opens. */ -async function settledTurn() { - await host.restoreReadableSessions([SESSION]) - const { items } = await host.journalSnapshot(SESSION) +async function settledTurn(reader = host) { + await reader.restoreReadableSessions([SESSION]) + const { items } = await reader.journalSnapshot(SESSION) const turn = items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) const [settled] = [...selectStructuredAgentSettledTurns(items).values()] return { turn, settled } } +/** What the next launch reads: a host that has begun quitting opens no chat. */ +async function settledTurnAfterQuit() { + const state = hostTestState() + const reader = new StructuredAgentSessionHost({ + store: state.store, + adapter: adapter(), + journalDatabase: openTestJournalHostDatabase(state.root), + recoveryCapsule: new AgentSessionRecoveryCapsule(state.root), + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => HOST_TEST_NOW + }) + try { + return await settledTurn(reader) + } finally { + await reader.flushAllStreamedEvents() + } +} + function lastSummary(statuses: AgentSessionStatusEvent[]) { const last = statuses.at(-1) return last?.type === 'status' ? last.session : null @@ -243,7 +262,7 @@ describe('a turn cut short by closing its provider', () => { expect(statuses.findLast((event) => event.type === 'status')).toMatchObject({ session: { status: 'idle', turnOutcome: 'interruption' } }) - const { settled } = await settledTurn() + const { settled } = await settledTurnAfterQuit() expect( settled && describeNativeChatTurnStatus({ elapsedSeconds: 0, ...settled }) ).toMatchObject({ key: 'failedAfter' }) @@ -305,7 +324,7 @@ describe('a turn cut short by closing its provider', () => { await host.flushAllStreamedEvents({ trigger: 'quit' }) - const { turn } = await settledTurn() + const { turn } = await settledTurnAfterQuit() expect(turn).toMatchObject({ state: 'interrupted' }) expect(turn).not.toHaveProperty('outcome') }) From 4ecee3972646e20790fb11a808cd12936678b071 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:33:12 -0700 Subject: [PATCH 034/107] test(native-chat): the deadlock test settles chats a crash cut mid-turn, which startup selects --- ...ctured-agent-session-startup-state.test.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 7b97e7a4c1f5..76a60970319b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -14,6 +14,7 @@ import { readTestJournalSessionStatus } from '../agent-session-journal/journal-host-database-test-support' import Database from '../../sqlite/sync-database' +import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' import { editPersistedTestAgentSessionStore } from '../../runtime/agent-session-record-store-test-harness' @@ -52,6 +53,22 @@ async function crashMidSend(rig: RestTestRig, sessionId: string, listed = true): await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) } +/** A chat whose turn is running when Orca dies: startup selects it to settle. */ +async function crashMidTurn(rig: RestTestRig, sessionId: string): Promise { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + const [{ providerIdentity }] = await Promise.all( + rig.adapter.dispatch.mock.results.slice(-1).map((result) => result.value) + ) + await rig.host + .collaboratorsForTests() + .sessions.get(sessionId)! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } + ) +} + /** What startup runs, in order, on the ids the tab list names. */ async function startup(rig: RestTestRig, listed: readonly string[]) { await rig.host.reconcileRestartLeases() @@ -456,10 +473,9 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', vi.spyOn(console, 'warn').mockImplementation(() => undefined) const rig = await newRig() for (const sessionId of ['session-a', 'session-b']) { - await crashMidSend(rig, sessionId) + await crashMidTurn(rig, sessionId) } await restTestChat(rig, 'session-healthy', { message: 'fine' }) - await rig.host.flushAllStreamedEvents() await rig.crash() const gate = new StructuredAgentSessionStartupGate(CEILING_MS) gate.hold() @@ -467,16 +483,17 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', const listed = listedIds(rig) await rig.host.reconcileRestartLeases() rig.host.seedStoredStatuses(listed) + expect(readUnsettledJournalSessionIds(db(rig)).toSorted()).toEqual(['session-a', 'session-b']) const started = Date.now() const elapsed = () => Date.now() - started // Before the settle: a send to a crashed chat, and the option read a chat pane fires on mount. const sendBefore = sendRestTestMessage(rig, 'session-a', 'during startup').then(elapsed) - const optionsBefore = rig.host.readOptions('session-a').then(elapsed, elapsed) + const optionsBefore = rig.host.readOptions('session-a').then(elapsed) const settled = rig.host.settleOwedSessions(listed) gate.openWhen(settled) // During the settle: the second crashed chat, which the settle has not reached yet. - const optionsDuring = rig.host.readOptions('session-b').then(elapsed, elapsed) + const optionsDuring = rig.host.readOptions('session-b').then(elapsed) const healthyRead = rig.host.journalSnapshot('session-healthy').then(elapsed) const settleEnded = await settled.then(elapsed) From 33fc782b55fa21676af1d7e0434adf715acd183a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:40:57 -0700 Subject: [PATCH 035/107] fix(native-chat): type the fold undo's containers and the restore test's stand-in host --- .../journal-fold-undo.ts | 14 ++--- .../journal-session-state-test-writes.ts | 5 +- ...runtime-structured-session-restore.test.ts | 58 +++++++++---------- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts index 9021a6d2da22..4716f59334ce 100644 --- a/src/main/native-chat/agent-session-journal/journal-fold-undo.ts +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts @@ -62,7 +62,7 @@ export type JournalFoldUndo = { /** Null when the fold's containers cannot record their changes; the caller re-reads it instead. */ export function beginJournalFoldUndo(state: JournalReducerState): JournalFoldUndo | null { - const containers = [ + const containers: readonly object[] = [ state.items, state.itemFences, state.tombstones, @@ -71,12 +71,12 @@ export function beginJournalFoldUndo(state: JournalReducerState): JournalFoldUnd state.aliases, state.appliedSettlementIds ] - const recording = containers.filter( - (container): container is JournalFoldMap | JournalFoldSet => - container instanceof JournalFoldMap || container instanceof JournalFoldSet - ) - if (recording.length !== containers.length) { - return null + const recording: { onChange: ((undo: Undo) => void) | null }[] = [] + for (const container of containers) { + if (!(container instanceof JournalFoldMap || container instanceof JournalFoldSet)) { + return null + } + recording.push(container) } const undos: Undo[] = [] const record = (undo: Undo) => { diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts index 21968f1b71db..e0980135340a 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts @@ -3,7 +3,8 @@ import { AGENT_JOURNAL_THREAD_SCOPE, type AgentJournalItemBody, - type AgentJournalItemIdentity + type AgentJournalItemIdentity, + type AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournal } from './journal-store' @@ -65,7 +66,7 @@ export async function settledTurn(journal: AgentSessionJournal, turnId: string): }) } -export function userMessage(text: string): AgentJournalItemBody { +export function userMessage(text: string): AgentJournalMessageItem { return { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] } } diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 9e8ca9a83a7c..f468807ab8eb 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -1,9 +1,21 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' +import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { OrcaRuntimeService } from './orca-runtime' afterEach(() => setStructuredAgentSessionHost(null)) +/** A host offering the startup step's seed and settle, plus the members a test drives. */ +function installStartupHost(members: object): void { + const host = { + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined, + ...members + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: startup restoration reaches only the members each test supplies. + setStructuredAgentSessionHost(host as unknown as StructuredAgentSessionHost) +} + describe('structured session cold restoration', () => { it('skips every heavy recovery step when no durable session store exists', async () => { const runtime = new OrcaRuntimeService() @@ -18,11 +30,9 @@ describe('structured session cold restoration', () => { internal.hasPersistedStructuredAgentSessionStore = () => false internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ - reconcileRestartLeases, - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined - } as never) + installStartupHost({ + reconcileRestartLeases + }) await runtime.prepareStructuredAgentSessionStartupRestoration() @@ -45,12 +55,10 @@ describe('structured session cold restoration', () => { internal.hasPersistedStructuredAgentSessionStore = () => true internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases, - restoreReadableSessions, - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined - } as never) + restoreReadableSessions + }) await runtime.prepareStructuredAgentSessionStartupRestoration() @@ -82,13 +90,11 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = hydrate internal.refreshMobileSessionPtyRecords = refresh internal.ensureStructuredAgentSessionHost = ensureHost - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases, - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined, restoreReadableSessions, listSessionTabs: () => [] - } as never) + }) const first = runtime.restoreStructuredAgentSessionTabs() const second = runtime.restoreStructuredAgentSessionTabs() @@ -151,17 +157,15 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined, getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: ['session-survives-rollback'] }), restoreReadableSessions, listSessionTabs: () => [] - } as never) + }) await runtime.restoreStructuredAgentSessionTabs() runtime.startStructuredAgentSessionHistoryRestore() @@ -213,14 +217,12 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined, getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: [] }), restoreReadableSessions, listSessionTabs: () => [] - } as never) + }) await runtime.restoreStructuredAgentSessionTabs() runtime.startStructuredAgentSessionHistoryRestore() @@ -249,10 +251,8 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined, restoreReadableSessions: async () => undefined, close: closeStructuredSession, setSessionTabVisibility, @@ -263,7 +263,7 @@ describe('structured session cold restoration', () => { agent: 'codex' } ] - } as never) + }) runtime.syncWindowGraph(1, { tabs: [], leaves: [], @@ -366,10 +366,8 @@ describe('structured session cold restoration', () => { internal.hydrateHeadlessMobileSessionTabsFromWorkspaceSession = () => new Set() internal.refreshMobileSessionPtyRecords = async () => new Set() internal.ensureStructuredAgentSessionHost = async () => undefined - setStructuredAgentSessionHost({ + installStartupHost({ reconcileRestartLeases: async () => undefined, - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined, restoreReadableSessions: async () => undefined, listSessionTabs: () => [ { @@ -378,7 +376,7 @@ describe('structured session cold restoration', () => { agent: 'claude' } ] - } as never) + }) await runtime.restoreStructuredAgentSessionTabs() From 247d22458162da94316e332614455bd538ae1874 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:52:41 -0700 Subject: [PATCH 036/107] refactor(native-chat): one rule decides which chats this host can settle; boot drops the rest for good --- .../structured-agent-session-provider-support.ts | 12 ++++++++++++ .../structured-agent-session-reveal.ts | 9 +++++++-- .../structured-agent-session-startup-state.test.ts | 11 ++++++++++- .../structured-agent-session-startup-state.ts | 9 +++++---- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-support.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-support.ts index 15af150c12f1..8d70d38b1a05 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-support.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-support.ts @@ -41,3 +41,15 @@ export function adapterSupportsRecord( // Old Codex records stay readable unless the adapter explicitly rejects their location. return record.provider === 'codex' && (adapter.supportsLocation?.(record.location) ?? true) } + +/** + * Whether this host can settle a chat: its record is present and this host runs its provider. + * Every settle of a closed chat, startup's choice of stored statuses to drop, and any writer of a + * missing status use this one rule, so none writes a row another drops on every launch. + */ +export function hostCanSettleRecord( + adapter: StructuredAgentSessionAdapter, + record: AgentSessionRecord | null +): record is AgentSessionRecord { + return record !== null && adapterSupportsRecord(adapter, record) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 029d4aa16de3..7fb333a9a860 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -13,7 +13,10 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { sessionTabListed } from './structured-agent-session-host-tabs' -import { adapterSupportsRecord } from './structured-agent-session-provider-support' +import { + adapterSupportsRecord, + hostCanSettleRecord +} from './structured-agent-session-provider-support' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { @@ -88,6 +91,8 @@ export function createStructuredAgentSessionHostRestore( const failures = reportEachFailureOnce(deps.onLeaseReconcileFailure) const reconcile = createReaderReconcile(reconcileLeases, failures) const supportsRecord = (record: AgentSessionRecord) => adapterSupportsRecord(deps.adapter, record) + const canSettle = (record: AgentSessionRecord | null): record is AgentSessionRecord => + hostCanSettleRecord(deps.adapter, record) const readRestore: StructuredAgentSessionReadRestoreDeps = { openDeps: deps, isListed: (sessionId) => sessionTabListed(deps.store, sessionId), @@ -112,7 +117,7 @@ export function createStructuredAgentSessionHostRestore( restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), ...createStructuredAgentSessionStartupState({ openDeps: deps, - supportsRecord, + canSettle, seedStatus, resolveRecovery: readRestore.resolveRecovery, restoreListed: (records) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 76a60970319b..19e7cb6b8440 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -507,7 +507,7 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', }) describe('a stored status no settle here can clear (R2A-4)', () => { - it('drops the row of a chat whose record is gone or whose provider this host does not serve', async () => { + it('drops the row of a chat whose record is gone or whose provider this host does not serve, so the next boot selects neither', async () => { const rig = await newRig() await crashMidSend(rig, 'session-gone', false) await crashMidSend(rig, 'session-elsewhere', false) @@ -524,5 +524,14 @@ describe('a stored status no settle here can clear (R2A-4)', () => { expect(readTestJournalSessionStatus(rig.root, sessionId)).toBeNull() } expect(opened(rig, ['session-gone', 'session-elsewhere'])).toEqual([]) + + // The obligation died: the next boot finds no row to select and opens neither chat. + await rig.crash() + await rig.boot() + await startup(rig, listedIds(rig)) + for (const sessionId of ['session-gone', 'session-elsewhere']) { + expect(readTestJournalSessionStatus(rig.root, sessionId)).toBeNull() + } + expect(opened(rig, ['session-gone', 'session-elsewhere'])).toEqual([]) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 945e895eb0b7..654d7eda78b8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -35,7 +35,8 @@ export type StructuredAgentSessionStartupStateDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { store: Pick } - supportsRecord: (record: AgentSessionRecord) => boolean + /** `hostCanSettleRecord` bound to this host's adapter. */ + canSettle: (record: AgentSessionRecord | null) => record is AgentSessionRecord seedStatus: ( record: AgentSessionRecord, stored: { projected: StructuredAgentSessionStatusProjection; lastActivityAt: number } @@ -96,7 +97,7 @@ function seedStoredStatuses( background.push(sessionId) continue } - if (!deps.supportsRecord(record)) { + if (!deps.canSettle(record)) { continue } if (!byId.has(sessionId)) { @@ -136,7 +137,7 @@ async function settleOwedSessions( const others: AgentSessionRecord[] = [] for (const sessionId of readUnsettledJournalSessionIds(database.db)) { const record = deps.openDeps.store.getRecord(sessionId) - if (!record || !deps.supportsRecord(record)) { + if (!deps.canSettle(record)) { dropUnreachableStatus(database, sessionId, record) continue } @@ -212,7 +213,7 @@ async function settleClosed( deps: StructuredAgentSessionStartupStateDeps, record: AgentSessionRecord ): Promise { - if (deps.isDisposed() || deps.hasSession(record.sessionId)) { + if (deps.isDisposed() || deps.hasSession(record.sessionId) || !deps.canSettle(record)) { return } const opened = await openStructuredAgentSessionConversationJournal(deps.openDeps, record, { From a7c9b3f0e3919ac4f4e874e390b1f94c99a1e2b2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:05:00 -0700 Subject: [PATCH 037/107] refactor(native-chat): expose the one closed-chat settle; it refuses a chat this host cannot settle --- ...ctured-agent-session-startup-state.test.ts | 42 +++++++++++++++++++ .../structured-agent-session-startup-state.ts | 11 +++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 19e7cb6b8440..59147622b43d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -14,6 +14,12 @@ import { readTestJournalSessionStatus } from '../agent-session-journal/journal-host-database-test-support' import Database from '../../sqlite/sync-database' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' +import { + createStructuredAgentSessionStartupState, + type StructuredAgentSessionStartupStateDeps +} from './structured-agent-session-startup-state' import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' @@ -507,6 +513,42 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', }) describe('a stored status no settle here can clear (R2A-4)', () => { + it('is refused by the shared closed-chat settle, which opens nothing for it', async () => { + const opens: string[] = [] + const openDeps = new Proxy( + {}, + { + get: (_target, key) => { + opens.push(String(key)) + throw new Error('the settle opened a chat this host cannot settle') + } + } + ) + const refused: AgentSessionRecord[] = [] + const canSettle = (record: AgentSessionRecord | null): record is AgentSessionRecord => { + if (record) { + refused.push(record) + } + return false + } + const state = createStructuredAgentSessionStartupState({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a refused settle reads no open dependency; any read throws and fails the test. + openDeps: openDeps as StructuredAgentSessionStartupStateDeps['openDeps'], + canSettle, + seedStatus: vi.fn(), + resolveRecovery: vi.fn(async () => true), + restoreListed: vi.fn(async () => undefined), + serialize: (_sessionId, task) => task(), + hasSession: () => false, + isDisposed: () => false + }) + const record = agentSessionRecordFixture() + + await expect(state.settleClosedChat(record)).resolves.toBe(false) + expect(refused).toEqual([record]) + expect(opens).toEqual([]) + }) + it('drops the row of a chat whose record is gone or whose provider this host does not serve, so the next boot selects neither', async () => { const rig = await newRig() await crashMidSend(rig, 'session-gone', false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 654d7eda78b8..a95cfa2522fe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -56,6 +56,9 @@ export type StructuredAgentSessionStartupState = { seedStoredStatuses: (listedIds: readonly string[]) => string[] /** Settles every chat a gone process left with work, once per host. Never rejects. */ settleOwedSessions: (listedIds: readonly string[]) => Promise + /** The one settle of a chat nothing holds open, for any caller inside the chat's serialize. + * False, opening nothing, for a chat `canSettle` rejects, one already open, or after quit. */ + settleClosedChat: (record: AgentSessionRecord) => Promise } export function createStructuredAgentSessionStartupState( @@ -67,7 +70,8 @@ export function createStructuredAgentSessionStartupState( settleOwedSessions: (listedIds) => { settling ??= settleOwedSessions(deps, listedIds) return settling - } + }, + settleClosedChat: (record) => settleClosed(deps, record) } } @@ -212,12 +216,13 @@ async function resolveRecoveringLeases(deps: StructuredAgentSessionStartupStateD async function settleClosed( deps: StructuredAgentSessionStartupStateDeps, record: AgentSessionRecord -): Promise { +): Promise { if (deps.isDisposed() || deps.hasSession(record.sessionId) || !deps.canSettle(record)) { - return + return false } const opened = await openStructuredAgentSessionConversationJournal(deps.openDeps, record, { deferPerSessionImport: true }) await opened.session.journal.close() + return true } From e4db1da60dda555a63fe38e3d93ede6e6ed57d74 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:11:37 -0700 Subject: [PATCH 038/107] test(native-chat): type the restore test's stand-in host members by the host's own members --- ...rca-runtime-structured-session-restore.test.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index f468807ab8eb..acd6524ca6f2 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -5,8 +5,21 @@ import { OrcaRuntimeService } from './orca-runtime' afterEach(() => setStructuredAgentSessionHost(null)) +/** The host members startup restoration reaches, beyond the seed and settle every test gets. */ +type StartupHostMembers = Partial< + Pick< + StructuredAgentSessionHost, + | 'reconcileRestartLeases' + | 'restoreReadableSessions' + | 'listSessionTabs' + | 'getPersistedVisibleSessionTabIndex' + | 'setSessionTabVisibility' + | 'close' + > +> + /** A host offering the startup step's seed and settle, plus the members a test drives. */ -function installStartupHost(members: object): void { +function installStartupHost(members: StartupHostMembers): void { const host = { seedStoredStatuses: (ids: readonly string[]) => [...ids], settleOwedSessions: async () => undefined, From aae0428ef334f8641206a3d9343a69f43978646f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:16:12 -0700 Subject: [PATCH 039/107] test(native-chat): the legacy record import expects the current schema version by name --- .../agent-session-legacy-record-import.test.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/runtime/agent-session-legacy-record-import.test.ts b/src/main/runtime/agent-session-legacy-record-import.test.ts index da09cb2257d8..cd2283b05154 100644 --- a/src/main/runtime/agent-session-legacy-record-import.test.ts +++ b/src/main/runtime/agent-session-legacy-record-import.test.ts @@ -16,6 +16,10 @@ import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' import { journalPragmaNumber } from '../native-chat/agent-session-journal/journal-database' +import { + JOURNAL_DB_OLDEST_RELEASED_VERSION, + JOURNAL_DB_SCHEMA_VERSION +} from '../native-chat/agent-session-journal/journal-database-schema' import type { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' import type { LegacyAgentSessionRecordImportReport } from './agent-session-legacy-record-import' import { AgentSessionRecordStore } from './agent-session-record-store' @@ -134,7 +138,7 @@ describe('copying the records file into the chat database', () => { const { database, store, reports } = await install() expect(reports).toEqual([]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(4) + expect(journalPragmaNumber(database.db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(store.getRecord(ALPHA)).toEqual(unreconciled(alpha)) expect(store.getRecord(BETA)).toEqual(unreconciled(beta)) expect(store.listOperationRows()).toEqual([OPERATION]) @@ -171,7 +175,7 @@ describe('copying the records file into the chat database', () => { error: expect.objectContaining({ message: 'agent_session_store_corrupt' }) } ]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(4) + expect(journalPragmaNumber(database.db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(store.listRecords()).toEqual([]) expect(await readFile(legacyPath(), 'utf-8')).toBe(adHoc) }) @@ -308,7 +312,9 @@ describe('a primary the backup stands in for', () => { const { database, store, reports } = await install() expect(reports).toMatchObject([{ kind: 'unavailable' }]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(3) + expect(journalPragmaNumber(database.db, 'user_version')).toBe( + JOURNAL_DB_OLDEST_RELEASED_VERSION + ) expect(store.listRecords()).toEqual([]) }) @@ -320,7 +326,9 @@ describe('a primary the backup stands in for', () => { const { database, store, reports } = await install() expect(reports).toMatchObject([{ kind: 'unavailable' }]) - expect(journalPragmaNumber(database.db, 'user_version')).toBe(3) + expect(journalPragmaNumber(database.db, 'user_version')).toBe( + JOURNAL_DB_OLDEST_RELEASED_VERSION + ) expect(store.listRecords()).toEqual([]) }) }) @@ -392,7 +400,7 @@ describe('the tab index from before the table', () => { const { database, store } = await install() - expect(journalPragmaNumber(database.db, 'user_version')).toBe(4) + expect(journalPragmaNumber(database.db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) expect(store.listRecords().map(({ sessionId }) => sessionId)).toEqual([BETA, ALPHA]) // Restore then takes the profile's tabs, beside the tab the owed-era chat left. expect(store.getVisibleSessionTabIndex()).toEqual({ present: false, sessionIds: [BETA] }) From 02bd7e43e28b7aaea320b50165e650e6f34f9471 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:20:20 -0700 Subject: [PATCH 040/107] fix(native-chat): a failed append that removed an entry re-reads the fold instead of re-adding it out of order --- .../journal-fold-undo.test.ts | 283 ++++++++++++++++++ .../journal-fold-undo.ts | 40 ++- .../journal-open-settlement-plan.ts | 2 +- .../journal-store-collaborators.ts | 10 +- 4 files changed, 317 insertions(+), 18 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts new file mode 100644 index 000000000000..2074e70cae69 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts @@ -0,0 +1,283 @@ +// A failed append's undo, as a property: every append first runs with a failing COMMIT, then for +// real, and after each the in-memory fold equals a fresh replay of the disk, container by container +// with key order, scalars and turn scope, and the stored status equals a fresh derivation. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { + AgentJournalItemIdentity, + AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { + createTrackedJournalOpener, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import { JournalQueuedMessages } from './journal-queued-messages' +import { MAX_JOURNAL_APPLIED_SETTLEMENT_IDS, type JournalReducerState } from './journal-reducer' +import { deriveJournalSessionStatus } from './journal-session-state' +import { + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS +} from './journal-session-state-test-corpus' +import type { AgentSessionJournal } from './journal-store' + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-fold-undo-')) + clock = 1_000 +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +function identity(sessionId: string): AgentSessionJournalIdentity { + return { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + } +} + +function open(sessionId: string): Promise { + return journals.open({ + identity: identity(sessionId), + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}-${clock}`, + currentFence: () => CORPUS_FENCE + }) +} + +function failNextCommit(): () => void { + const connection = openTestJournalHostDatabase(root).db + const exec = connection.exec.bind(connection) + let failing = true + const spy = vi.spyOn(connection, 'exec').mockImplementation((sql: string) => { + if (failing && sql === 'COMMIT') { + failing = false + throw new Error('COMMIT failed: disk I/O error') + } + return exec(sql) + }) + return () => spy.mockRestore() +} + +type JournalInternals = { + fold: JournalReducerState + rowWriter: { enqueue: (build: unknown, hook?: unknown) => Promise } +} + +function internals(journal: AgentSessionJournal): JournalInternals { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the store's private fold and row writer, read to compare against a replay; both fields exist on every store. + return journal as unknown as JournalInternals +} + +/** Every scalar, every container in key order, and the turn scope's open turn. */ +function comparableFold(state: JournalReducerState) { + const { derivedTurnScope, ...rest } = state + return { + ...Object.fromEntries( + Object.entries(rest).map(([key, value]) => [ + key, + value instanceof Map || value instanceof Set ? [...value.entries()] : value + ]) + ), + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the scope's private open turn, the one field a clone carries. + openTurn: (derivedTurnScope as unknown as { openTurnItemId: string | null }).openTurnItemId + } +} + +/** The fold as the store serves it: a stale fold is re-read by the next public read first. */ +function servedFold(journal: AgentSessionJournal): JournalReducerState { + journal.cursor() + return internals(journal).fold +} + +function expectFoldIsTheDisk(journal: AgentSessionJournal, sessionId: string): void { + const loaded = loadTestJournal(root, sessionId)! + expect(comparableFold(servedFold(journal))).toEqual(comparableFold(loaded.state)) + expect(readTestJournalSessionStatus(root, sessionId)).toEqual( + deriveJournalSessionStatus(loaded.state, { + settlesRosters: !loaded.corrupt, + currentFence: CORPUS_FENCE + }) + ) +} + +/** Every append is first tried with a failing COMMIT, then for real, one append at a time. + * Answers every check that failed, so a case that swallows an append's error still reports it. */ +function failEveryAppendOnce(journal: AgentSessionJournal, sessionId: string): unknown[] { + const violations: unknown[] = [] + const check = (run: () => void) => { + try { + run() + } catch (error) { + violations.push(error) + throw error + } + } + const writer = internals(journal).rowWriter + const enqueue = writer.enqueue.bind(writer) + let tail: Promise = Promise.resolve() + const twice = async (build: unknown, hook?: unknown) => { + const statusBefore = readTestJournalSessionStatus(root, sessionId) + const restore = failNextCommit() + const failed = await enqueue(build, hook).then( + () => new Error('the failing COMMIT committed'), + (error: unknown) => error + ) + restore() + check(() => { + expect(readTestJournalSessionStatus(root, sessionId)).toEqual(statusBefore) + expectFoldIsTheDisk(journal, sessionId) + }) + if (!String(failed).includes('COMMIT failed')) { + // Refused before its COMMIT (a revision rule, a held id): nothing to retry. + throw failed + } + const committed = await enqueue(build, hook) + check(() => expectFoldIsTheDisk(journal, sessionId)) + return committed + } + writer.enqueue = (build, hook) => { + const run = tail.then(() => twice(build, hook)) + tail = run.catch(() => undefined) + return run + } + return violations +} + +function assistant(text: string) { + return { + kind: 'message' as const, + role: 'assistant' as const, + blocks: [{ type: 'text' as const, text }] + } +} + +function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-corpus', turnId, ordinal } +} + +const THREAD = { fence: CORPUS_FENCE, turnScope: { kind: 'thread' as const } } + +describe('a failed append leaves the fold equal to a replay of the disk', () => { + it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { + const journal = await open(name) + const violations = failEveryAppendOnce(journal, name) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + expect(violations).toEqual([]) + }) + + it('through every case on one chat, a tombstone and a recreate', async () => { + const journal = await open('mixed') + const violations = failEveryAppendOnce(journal, 'mixed') + for (const name of JOURNAL_SESSION_STATE_CASES) { + // Cases reuse ids; a revision rule refusing a stale one is a write path too. + await JOURNAL_SESSION_STATE_CORPUS[name](journal).catch(() => undefined) + } + const id = codexItem('t-x', 0) + await journal.appendItem(id, assistant('a'), THREAD) + await journal.appendTombstone(id, { fence: CORPUS_FENCE }) + await journal.appendItem(id, assistant('b'), THREAD) + expect(violations).toEqual([]) + }) + + it('keeps map order when the failed append tombstoned an item that is not the newest', async () => { + const journal = await open('order') + for (const ordinal of [0, 1]) { + await journal.appendItem(codexItem('t-1', ordinal), assistant('x'), THREAD) + } + const violations = failEveryAppendOnce(journal, 'order') + + await journal.appendTombstone(codexItem('t-1', 0), { fence: CORPUS_FENCE }) + expect(violations).toEqual([]) + }) + + it.each(JOURNAL_SESSION_STATE_CASES)( + 'with a bookkeeping savepoint that always fails: %s', + async (name) => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.spyOn(JournalQueuedMessages.prototype, 'onRowInTransaction').mockImplementation((db) => { + db.prepare( + `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, + queued_sends, live_child_work, summary_json, last_activity_at) + VALUES ('savepoint-write', 'running', NULL, 0, 0, 0, '{}', 0)` + ).run() + throw new Error('bookkeeping failed') + }) + const journal = await open(name) + const violations = failEveryAppendOnce(journal, name) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + expect(violations).toEqual([]) + + const savepointWrite = openTestJournalHostDatabase(root) + .db.prepare("SELECT 1 FROM journal_session_state WHERE session_id = 'savepoint-write'") + .get() + expect(savepointWrite).toBeUndefined() + } + ) +}) + +describe('a failed append that evicted the oldest settlement id (R3W-1)', () => { + it('forgets the same ids a replay forgets, so a re-sent evicted settle is skipped live too', async () => { + const journal = await open('cap') + const note = (id: string) => ({ + kind: 'item' as const, + identity: { provider: 'orca' as const, clientMessageId: id }, + body: { kind: 'status' as const, text: id }, + turnScope: { kind: 'thread' as const } + }) + for (let index = 0; index < MAX_JOURNAL_APPLIED_SETTLEMENT_IDS; index += 1) { + await journal.appendLifecycleBatch({ + settlementId: `s-${index}`, + fence: CORPUS_FENCE, + mutations: [note(`n-${index}`)] + }) + } + const restore = failNextCommit() + await expect( + journal.appendLifecycleBatch({ + settlementId: 's-new', + fence: CORPUS_FENCE, + mutations: [note('n-new')] + }) + ).rejects.toThrow('COMMIT failed') + restore() + expectFoldIsTheDisk(journal, 'cap') + + // One more commit evicts the oldest, as a replay does; then a settle re-sent under that id. + await journal.appendLifecycleBatch({ + settlementId: 's-after', + fence: CORPUS_FENCE, + mutations: [note('n-after')] + }) + await journal.appendLifecycleBatch({ + settlementId: 's-1', + fence: CORPUS_FENCE, + mutations: [ + { + kind: 'item', + identity: codexItem('turn-1', 7), + body: { kind: 'tool-call', name: 'shell', input: { command: 'x' }, state: 'running' }, + turnScope: { kind: 'thread' } + } + ] + }) + + expectFoldIsTheDisk(journal, 'cap') + }, 120_000) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts index 4716f59334ce..e18d8f4b28de 100644 --- a/src/main/native-chat/agent-session-journal/journal-fold-undo.ts +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.ts @@ -1,15 +1,19 @@ // One append's undo of the fold. An append folds its row inside its transaction, so the status -// written with it describes it; if the transaction fails, exactly what the row touched is put back. -// Entries are replaced on change, never edited, so restoring each touched key restores the fold, -// and an entry a reader holds is never one the failed row changed. +// written with it describes it; if the transaction fails, what the row touched is put back. +// Entries are replaced on change, never edited, so an entry a reader holds is never one the failed +// row changed. A replaced or added key goes back in place; a removed one cannot, since re-adding it +// puts it last and key order is part of the fold (eviction is oldest-first, ties go to map order), +// so a unit that removed an entry declines the undo and its caller re-reads the fold instead. import type { JournalReducerState } from './journal-reducer' type Undo = () => void +/** Null for a change no undo can put back in place: the removal of an entry. */ +type Recorder = (undo: Undo | null) => void /** A fold container that, while a unit is open, tells it each change it is about to make. */ export class JournalFoldMap extends Map { - onChange: ((undo: Undo) => void) | null = null + onChange: Recorder | null = null override set(key: K, value: V): this { this.record(key) @@ -17,7 +21,9 @@ export class JournalFoldMap extends Map { } override delete(key: K): boolean { - this.record(key) + if (this.onChange && super.has(key)) { + this.onChange(null) + } return super.delete(key) } @@ -36,7 +42,7 @@ export class JournalFoldMap extends Map { } export class JournalFoldSet extends Set { - onChange: ((undo: Undo) => void) | null = null + onChange: Recorder | null = null override add(value: T): this { if (this.onChange && !super.has(value)) { @@ -47,7 +53,7 @@ export class JournalFoldSet extends Set { override delete(value: T): boolean { if (this.onChange && super.has(value)) { - this.onChange(() => super.add(value)) + this.onChange(null) } return super.delete(value) } @@ -56,8 +62,9 @@ export class JournalFoldSet extends Set { export type JournalFoldUndo = { /** COMMIT landed: the fold as it stands is adopted. */ commit: () => void - /** The transaction failed: every change since the unit began is put back, newest first. */ - rollback: () => void + /** The transaction failed: every change since the unit began is put back, newest first. False, + * changing nothing, when the unit removed an entry; the caller then re-reads the fold. */ + rollback: () => boolean } /** Null when the fold's containers cannot record their changes; the caller re-reads it instead. */ @@ -71,7 +78,7 @@ export function beginJournalFoldUndo(state: JournalReducerState): JournalFoldUnd state.aliases, state.appliedSettlementIds ] - const recording: { onChange: ((undo: Undo) => void) | null }[] = [] + const recording: { onChange: Recorder | null }[] = [] for (const container of containers) { if (!(container instanceof JournalFoldMap || container instanceof JournalFoldSet)) { return null @@ -79,8 +86,13 @@ export function beginJournalFoldUndo(state: JournalReducerState): JournalFoldUnd recording.push(container) } const undos: Undo[] = [] - const record = (undo: Undo) => { - undos.push(undo) + let restorable = true + const record: Recorder = (undo) => { + if (undo) { + undos.push(undo) + } else { + restorable = false + } } const scalars = { epoch: state.epoch, @@ -103,11 +115,15 @@ export function beginJournalFoldUndo(state: JournalReducerState): JournalFoldUnd commit: detach, rollback: () => { detach() + if (!restorable) { + return false + } for (const undo of undos.toReversed()) { undo() } Object.assign(state, scalars) state.derivedTurnScope = derivedTurnScope + return true } } } diff --git a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts index 58558c63b339..a846f9023beb 100644 --- a/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts +++ b/src/main/native-chat/agent-session-journal/journal-open-settlement-plan.ts @@ -80,7 +80,7 @@ type ItemFacts = { } // Per item object: a write replaces the entry it changes and never edits one, so an object's facts -// never change, and each append judges only the items it touched. +// never change. Each append still visits every item, but computes facts only for new objects. const ITEM_FACTS = new WeakMap() function itemFacts(item: AgentJournalRenderItem): ItemFacts { diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 487571107270..f36644b2238f 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -237,14 +237,14 @@ function backfillSessionStatus( /** * An append whose transaction failed after its row was folded: the undo puts back what the row - * changed. If it cannot, the fold is marked stale and folded again from what committed before its - * next use (the host database rolls a stranded transaction back before it hands out the connection, - * so that re-read never sees the failed row). + * changed. If it cannot (it throws, or the row removed an entry, which no undo puts back in its + * place), the fold is marked stale and folded again from what committed before its next use (the + * host database rolls a stranded transaction back before it hands out the connection, so that + * re-read never sees the failed row). */ function recoverJournalFold(host: JournalStoreHost, undo: JournalFoldUndo | null): void { try { - if (undo) { - undo.rollback() + if (undo?.rollback()) { return } } catch (error) { From c76f31aaf68fc37887c3448a38c1eac98d8f7004 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:36:19 -0700 Subject: [PATCH 041/107] refactor(native-chat): one command entry waits for the startup settle; the step starts before the first window's services --- ...ured-agent-session-attach-orchestration.ts | 15 +--- .../structured-agent-session-command-entry.ts | 23 ++++++ ...red-agent-session-conversation-lifetime.ts | 5 +- ...structured-agent-session-host-mutations.ts | 36 ++++----- .../structured-agent-session-options-read.ts | 5 +- .../structured-agent-session-rewind.ts | 3 +- ...ctured-agent-session-startup-state.test.ts | 77 +++++++++++++++++++ .../structured-agent-session-startup-state.ts | 9 ++- .../structured-conversation-command.ts | 3 +- ...structured-agent-session-create-support.ts | 52 ++++++++++--- src/main/runtime/orca-runtime-runtime-id.ts | 1 + ...runtime-structured-session-restore.test.ts | 5 +- ...structured-agent-session-queued-methods.ts | 11 ++- ...d-agent-session-send-compatibility.test.ts | 23 ++++++ .../structured-agent-session-thread-goal.ts | 5 +- .../structured-agent-session-startup-gate.ts | 70 ++++++++++++++--- ...-agent-session-startup-restoration.test.ts | 27 ++++++- ...-process-runtime-launch-activation.test.ts | 5 +- .../startup/main-process-runtime-launch.ts | 10 ++- 19 files changed, 309 insertions(+), 76 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-command-entry.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index 1e2e09621773..55437daf74f1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -41,6 +41,7 @@ import { withAgentSessionSpan, type AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' export type StructuredAgentSessionAttachOptions = { recordPhase?: AgentSessionCreatePhaseRecorder @@ -66,18 +67,6 @@ export function attachStructuredAgentSessionUnderSerialize( return context.tasks.trackAttach(runAttach(context, callerKey, params, options)) } -/** Startup's settle first, before the chat's lock, which the settle takes too; queued at once when - * startup is done, so attaches keep the order they arrived in. */ -function waitThenSerialize( - context: StructuredAgentSessionAttachContext, - sessionId: string, - task: () => Promise -): Promise { - const startup = context.deps.commandsReady?.() - const run = () => context.serialize(sessionId, task) - return startup ? startup.then(run) : run() -} - export function attachStructuredAgentSession( context: StructuredAgentSessionAttachContext, callerKey: string, @@ -88,7 +77,7 @@ export function attachStructuredAgentSession( // evicts, so no child is spawned behind the eviction and orphaned. const run = (recordPhase?: AgentSessionCreatePhaseRecorder) => context.tasks.trackAttach( - waitThenSerialize(context, sessionId, () => + serializeStructuredAgentSessionCommand(context, sessionId, () => runAttach(context, callerKey, params, { recordPhase }) ) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-command-entry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-command-entry.ts new file mode 100644 index 000000000000..388c9aae94ce --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-command-entry.ts @@ -0,0 +1,23 @@ +// The one way a chat command reaches a chat: host startup's settle first, then the chat's lock. +// The settle takes chats' locks too, so the wait comes before the lock, never under it. Nothing +// that holds a chat's lock calls this; /clear's attach of its replacement, under the source +// chat's lock, is the one call that does, and it can, because its own command already passed the +// wait and the wait never closes again. + +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' + +type CommandEntryContext = { + deps: Pick + serialize: (sessionId: string, task: () => Promise) => Promise +} + +/** Queued on the lock at once when startup is done, so commands keep the order they arrived in. */ +export function serializeStructuredAgentSessionCommand( + context: CommandEntryContext, + sessionId: string, + task: () => Promise +): Promise { + const startup = context.deps.commandsReady?.() + const run = () => context.serialize(sessionId, task) + return startup ? startup.then(run) : run() +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index d3071e9a8013..29f58ca42ff1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -103,8 +103,9 @@ export function createStructuredAgentSessionConversationLifetime(host: { * drop it after. */ conversation: async (sessionId: string): Promise => { - // Startup's settle first, before this chat's lock, which the settle takes too. No caller - // holds a chat's lock here. + // Reads (history, subscribe, snapshot, reveal) wait for startup's settle here, as commands do + // at `serializeStructuredAgentSessionCommand`: opening a crash-left chat would settle it ahead + // of startup's lease resolution. Before this chat's lock; no caller holds one here. const startup = deps().commandsReady?.() if (startup) { await startup diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index f0a52429d5d0..78e34312e5e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -56,6 +56,7 @@ import { readStructuredAgentSessionOptions, recordStructuredAgentSessionOptionIntent } from './structured-agent-session-options-read' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' export type StructuredAgentSessionMutationContext = { deps: StructuredAgentSessionHostDeps @@ -88,26 +89,21 @@ export function mutateStructuredAgentSession( plan: MutationPlan, prepareSession?: AgentSessionMutationRequest['prepareSession'] ): Promise> { - const run = () => - context.serialize(envelope.sessionId, () => - admitAndRunAgentSessionMutation({ - store: context.deps.store, - adapter: context.deps.adapter, - callerKey: caller.callerKey, - envelope, - plan, - journal: () => context.sessions.get(envelope.sessionId)?.journal, - prepareSession, - publish: (journal) => context.publish(envelope.sessionId, journal), - flushStreamedEvents: context.flushStreamedEvents, - providerChildPhase: () => context.sessions.get(envelope.sessionId)?.child?.phase, - now: () => context.now() - }) - ) - // Startup's settle first, before the chat's lock: the settle takes that lock too. Queued at once - // when startup is done, so commands keep the order they arrived in. - const startup = context.deps.commandsReady?.() - return startup ? startup.then(run) : run() + return serializeStructuredAgentSessionCommand(context, envelope.sessionId, () => + admitAndRunAgentSessionMutation({ + store: context.deps.store, + adapter: context.deps.adapter, + callerKey: caller.callerKey, + envelope, + plan, + journal: () => context.sessions.get(envelope.sessionId)?.journal, + prepareSession, + publish: (journal) => context.publish(envelope.sessionId, journal), + flushStreamedEvents: context.flushStreamedEvents, + providerChildPhase: () => context.sessions.get(envelope.sessionId)?.child?.phase, + now: () => context.now() + }) + ) } export function sendStructuredAgentSessionTurn( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts index 7b50ce346d6b..56937a11ddb0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-options-read.ts @@ -18,6 +18,7 @@ import { isCodexTurnOptionKey } from '../../codex/codex-structured-turn-start' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' type RestingOptions = Pick @@ -96,9 +97,7 @@ export async function readStructuredAgentSessionOptions( sessionId: string ): Promise { const { adapter, store } = context.deps - // Startup's settle first, before the chat's lock, which the settle takes too. - await context.deps.commandsReady?.() - const live = await context.serialize(sessionId, async () => { + const live = await serializeStructuredAgentSessionCommand(context, sessionId, async () => { const session = await context.openConversation(sessionId).catch((error: unknown) => { throw journalOpenReadRefusal(error) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index e855ae97ab27..d0186553fa83 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -23,6 +23,7 @@ import { conversationCommandBlocked } from './structured-conversation-command-ad import { rewindRefusal } from './structured-rewind-refusal' import { persistRewindRecord, recoverStructuredRewind } from './structured-rewind-recovery' import { mergeRetainedHostLifecycleRows } from './structured-rewind-retained-host-rows' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' export async function rewindStructuredAgentSession( context: StructuredAgentSessionMutationContext, @@ -32,7 +33,7 @@ export async function rewindStructuredAgentSession( ): Promise> { const { sessionId, clientOperationId } = params.envelope const store = context.deps.store - return context.serialize(sessionId, async () => { + return serializeStructuredAgentSessionCommand(context, sessionId, async () => { const result = await admitAndRunAgentSessionMutation({ store, adapter: context.deps.adapter, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 59147622b43d..0f63a0170abb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -6,6 +6,7 @@ import { cp, mkdir, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import { @@ -24,8 +25,10 @@ import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' import { editPersistedTestAgentSessionStore } from '../../runtime/agent-session-record-store-test-harness' +import { hostTestOperationId } from './structured-agent-session-host-test-data' import { createRestTestRig, + REST_TEST_CALLER, latestRestTestStatus, restTestChat, restTestOpens, @@ -510,8 +513,82 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', } expect(gate.ready()).toBeNull() }, 20_000) + + it('holds a rewind and a /clear sent straight to the host, and a read, then answers them as the settle ends', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const rig = await newRig() + const crashed = ['session-a', 'session-b', 'session-c'] + for (const sessionId of crashed) { + await crashMidTurn(rig, sessionId) + } + await rig.crash() + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + await rig.boot({ commandsReady: gate.ready }) + const listed = listedIds(rig) + await rig.host.reconcileRestartLeases() + rig.host.seedStoredStatuses(listed) + const opensAtBoot = crashed.map((id) => restTestOpens(rig, id)) + const started = Date.now() + const elapsed = () => Date.now() - started + + // Neither takes the RPC's reveal first; /clear is on the chat the settle reaches last. + const rewind = rig.host + .rewind(REST_TEST_CALLER, rewindParams('session-a', 'missing-item')) + .then(elapsed, elapsed) + const clear = rig.host + .conversationCommand(REST_TEST_CALLER, conversationCommandParams('session-b', 'clear')) + .then(elapsed, elapsed) + // A read opens a closed chat too, which would settle it ahead of the lease resolution. + const read = rig.host.journalSnapshot('session-c').then(elapsed) + await new Promise((resolve) => setTimeout(resolve, 20)) + // Held: none has opened its crashed chat ahead of the settle. + expect(crashed.map((id) => restTestOpens(rig, id))).toEqual(opensAtBoot) + const settled = rig.host.settleOwedSessions(listed) + gate.openWhen(settled) + const settleEnded = await settled.then(elapsed) + + expect(settleEnded).toBeLessThan(CEILING_MS / 2) + for (const at of await Promise.all([rewind, clear, read])) { + expect(at).toBeGreaterThanOrEqual(20) + expect(at).toBeLessThan(CEILING_MS / 2) + } + }, 20_000) }) +function rewindParams(sessionId: string, itemId: string) { + return { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.rewind', + sessionId, + fields: { itemId, expectedEpoch: 'epoch-unknown' } + }) + }, + itemId, + expectedEpoch: 'epoch-unknown' + } +} + +function conversationCommandParams(sessionId: string, command: 'clear') { + return { + envelope: { + sessionId, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.conversationCommand', + sessionId, + fields: { command } + }) + }, + command + } +} + describe('a stored status no settle here can clear (R2A-4)', () => { it('is refused by the shared closed-chat settle, which opens nothing for it', async () => { const opens: string[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index a95cfa2522fe..75169ebabc80 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -56,8 +56,13 @@ export type StructuredAgentSessionStartupState = { seedStoredStatuses: (listedIds: readonly string[]) => string[] /** Settles every chat a gone process left with work, once per host. Never rejects. */ settleOwedSessions: (listedIds: readonly string[]) => Promise - /** The one settle of a chat nothing holds open, for any caller inside the chat's serialize. - * False, opening nothing, for a chat `canSettle` rejects, one already open, or after quit. */ + /** + * The one settle of a chat nothing holds open. Call it inside the chat's serialize; it never + * waits on the startup gate. True when it opened and closed the chat, whether or not the chat + * had work (so a settled chat is opened again). False, opening nothing, when `canSettle` rejects + * the chat, the chat is already open, or after quit. Rejects when the open or the close throws, + * so callers must catch. + */ settleClosedChat: (record: AgentSessionRecord) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts index 065ead6e3433..98fdd5c52b8c 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts @@ -38,6 +38,7 @@ import { } from '../../../shared/agent-session-failure-words' import { structuredAgentSessionStartFailureFact } from './structured-agent-session-failure-text' import { carryQueuedMessagesToClearReplacement } from './structured-agent-session-queued-mutations' +import { serializeStructuredAgentSessionCommand } from './structured-agent-session-command-entry' /** A command's `error` is the sentence its row shows. */ export function conversationCommandFailure( @@ -147,7 +148,7 @@ export function runStructuredConversationCommand( ? record : null } - return context.serialize(sessionId, () => + return serializeStructuredAgentSessionCommand(context, sessionId, () => admitAndRunAgentSessionMutation({ store, adapter: context.deps.adapter, diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index cdd9b169e501..63dfbef92885 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -268,28 +268,57 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR } } + /** The tab restore's preparation: the startup step, then the terminal records refresh, which + * lists the daemon's terminals against the records the host build brought in. */ prepareStructuredAgentSessionStartupRestoration(): Promise { - this.structuredAgentSessionStartupRestorePromise ??= - this.prepareStructuredAgentSessionStartupRestorationOnce().catch((error) => { + this.structuredAgentSessionStartupRestorePromise ??= this.startStructuredAgentSessionStartup() + .then(async () => { + if (this.hasPersistedStructuredAgentSessionStore()) { + await this.refreshMobileSessionPtyRecords() + } + }) + .catch((error) => { this.structuredAgentSessionStartupRestorePromise = null throw error }) return this.structuredAgentSessionStartupRestorePromise } - protected async prepareStructuredAgentSessionStartupRestorationOnce(): Promise { + /** + * Desktop launch. The step needs neither the terminal daemon nor the hook server (structured + * chats never run in WSL, the lease check probes processes, and the seed publishes in process), + * so only the shell PATH is awaited, as before: the host build sets up the agents' launch + * environments from it. Off Windows it is already resolved. + */ + startStructuredAgentSessionStartupAfter(shellPathReady: Promise): void { + void shellPathReady + .then(() => this.startStructuredAgentSessionStartup()) + .catch((error: unknown) => console.warn('[structured-agent-session] startup failed', error)) + } + + /** The host's startup step, once: the host build, then the lease check, seed and settle. */ + startStructuredAgentSessionStartup(): Promise { + this.structuredAgentSessionStartupStepPromise ??= + this.startStructuredAgentSessionStartupOnce().catch((error) => { + this.structuredAgentSessionStartupStepPromise = null + throw error + }) + return this.structuredAgentSessionStartupStepPromise + } + + protected async startStructuredAgentSessionStartupOnce(): Promise { + const gate = this.structuredAgentSessionStartupGate + gate.stepStarted() // Chat commands held for startup go ahead however this ends: when the settle does, or now. - let settling = false + let opener: 'nothing to settle' | 'step failed' | null = 'nothing to settle' try { if (!this.hasPersistedStructuredAgentSessionStore()) { return } - // Durable agent records must exist before daemon inventory can be reconciled against them. // A refused host is no host: startup goes on, and only structured requests are refused. await ensureStructuredAgentSessionHostUnlessRefused(() => this.ensureStructuredAgentSessionHost() ) - await this.refreshMobileSessionPtyRecords() const host = getStructuredAgentSessionHost() if (host) { this.structuredAgentSessionBackgroundRestoreIds = @@ -297,14 +326,17 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR host, this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null, (settled) => { - settling = true - this.structuredAgentSessionStartupGate.openWhen(settled) + opener = null + gate.openWhen(settled) } ) } + } catch (error) { + opener &&= 'step failed' + throw error } finally { - if (!settling) { - this.structuredAgentSessionStartupGate.open() + if (opener) { + gate.open(opener) } } } diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index 70c3f0fdbeae..db39516ec6f9 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -129,6 +129,7 @@ export class OrcaRuntimeWithRuntimeId { protected structuredAgentSessionInventoryUnverifiable = false protected structuredAgentSessionStartupRestorePromise: Promise | null = null + protected structuredAgentSessionStartupStepPromise: Promise | null = null protected mobileSessionTabsChangeSequence = 0 diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index acd6524ca6f2..24587bb241b4 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -81,7 +81,7 @@ describe('structured session cold restoration', () => { expect(restoreReadableSessions).not.toHaveBeenCalled() }) - it('loads records, inventories PTYs, restores ownership, projects tabs, then restores history, once', async () => { + it('loads records, then restores ownership and inventories PTYs, projects tabs, then restores history, once', async () => { const runtime = new OrcaRuntimeService() const hydrate = vi.fn() const refresh = vi.fn(async () => new Set()) @@ -129,10 +129,11 @@ describe('structured session cold restoration', () => { expect(reconcileRestartLeases).toHaveBeenCalledOnce() expect(restoreReadableSessions).toHaveBeenCalledOnce() expect(ensureHost).toHaveBeenCalledOnce() + // The lease check probes processes, not terminals, so it does not wait for the PTY inventory. expect(ensureHost.mock.invocationCallOrder[0]).toBeLessThan( refresh.mock.invocationCallOrder[0] ?? Infinity ) - expect(refresh.mock.invocationCallOrder[0]).toBeLessThan( + expect(ensureHost.mock.invocationCallOrder[0]).toBeLessThan( reconcileRestartLeases.mock.invocationCallOrder[0] ?? Infinity ) expect(reconcileRestartLeases.mock.invocationCallOrder[0]).toBeLessThan( diff --git a/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts b/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts index a4b68ccbcd3a..038d2d6bb4bc 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-queued-methods.ts @@ -4,7 +4,7 @@ import { defineMethod } from '../core' import { - requireStructuredHost as requireHost, + requireInstalledStructuredHost as commandHost, structuredCallerFor as callerFor } from './structured-agent-session-gate' import { @@ -16,16 +16,19 @@ export const STRUCTURED_AGENT_SESSION_QUEUED_METHODS = [ defineMethod({ name: 'agentSession.queuedMessageSend', params: QueuedMessageActionParams, - handler: async (params, ctx) => requireHost(ctx).queuedMessageSend(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).queuedMessageSend(callerFor(ctx), params) }), defineMethod({ name: 'agentSession.queuedMessageDelete', params: QueuedMessageActionParams, - handler: async (params, ctx) => requireHost(ctx).queuedMessageDelete(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).queuedMessageDelete(callerFor(ctx), params) }), defineMethod({ name: 'agentSession.queuedMessagesResume', params: QueuedMessagesResumeParams, - handler: async (params, ctx) => requireHost(ctx).queuedMessagesResume(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).queuedMessagesResume(callerFor(ctx), params) }) ] diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts index 20de95a370d8..1d2621df01d1 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts @@ -11,6 +11,7 @@ import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/age import { call, clearStructuredHostStub, + envelope, hostCalls, installStructuredHostStub, SESSION, @@ -80,6 +81,28 @@ describe('a send that arrives before the host is built', () => { expect(response).not.toMatchObject({ ok: false }) expect(hostCalls.send).toHaveBeenCalledOnce() }) + + it.each([ + ['agentSession.queuedMessageSend', 'queuedMessageSend', { messageId: 'message-1' }], + ['agentSession.queuedMessageDelete', 'queuedMessageDelete', { messageId: 'message-1' }], + ['agentSession.queuedMessagesResume', 'queuedMessagesResume', {}], + ['agentSession.threadGoal', 'changeThreadGoal', { change: { kind: 'clear' } }] + ])('builds the host for %s too, and is answered by it', async (method, hostMethod, fields) => { + clearStructuredHostStub() + const answered = vi.fn(async () => ({ ok: true, replayed: false })) + const ensureStructuredAgentSessionHost = vi.fn(async () => { + installStructuredHostStub() + hostCalls[hostMethod] = answered + }) + + const response = await call(method, { envelope: envelope(), ...fields }, STRUCTURED_CLIENT, { + ensureStructuredAgentSessionHost + }) + + expect(ensureStructuredAgentSessionHost).toHaveBeenCalledOnce() + expect(response).not.toMatchObject({ ok: false }) + expect(answered).toHaveBeenCalledOnce() + }) }) function pendingSendResult() { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts b/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts index b28a51500d21..bde2f9ddd769 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-thread-goal.ts @@ -5,7 +5,7 @@ import { defineMethod } from '../core' import { - requireStructuredHost as requireHost, + requireInstalledStructuredHost as commandHost, structuredCallerFor as callerFor } from './structured-agent-session-gate' import { ThreadGoalParams } from './structured-agent-session-schemas' @@ -14,6 +14,7 @@ export const STRUCTURED_AGENT_SESSION_THREAD_GOAL_METHODS = [ defineMethod({ name: 'agentSession.threadGoal', params: ThreadGoalParams, - handler: async (params, ctx) => requireHost(ctx).changeThreadGoal(callerFor(ctx), params) + handler: async (params, ctx) => + (await commandHost(ctx)).changeThreadGoal(callerFor(ctx), params) }) ] diff --git a/src/main/runtime/structured-agent-session-startup-gate.ts b/src/main/runtime/structured-agent-session-startup-gate.ts index 6702f13bfa63..fbba5d3f6e9a 100644 --- a/src/main/runtime/structured-agent-session-startup-gate.ts +++ b/src/main/runtime/structured-agent-session-startup-gate.ts @@ -5,22 +5,39 @@ // those locks); none is refused. It always opens: when the settle ends, however it ends, or at a // ceiling, so a startup fault never strands a command. -// Under the shortest client request timeout (15 s): a held command lets go before its caller gives -// up, and an open settles its own chat, so letting go early is safe. +// Under the shortest client request timeout (15 s), counted from launch (`hold`): a held command +// lets go before its caller gives up, and an open settles its own chat, so letting go early is +// safe. Past it, the order is what is lost: a crashed turn may get its verdict from an open. const STARTUP_GATE_CEILING_MS = 10_000 +type StartupGateOpener = + | 'settle ended' + | 'settle failed' + | 'step failed' + | 'ceiling' + | 'nothing to settle' + export class StructuredAgentSessionStartupGate { private pending: Promise | null = null private release: (() => void) | null = null private ceiling: ReturnType | null = null + // One timing line per launch, for the startup measurements: written once the gate is open and + // the step is over (or never started). + private heldAt: number | null = null + private step: { startedAt: number; endedAt: number | null } | null = null + private opened: { at: number; by: StartupGateOpener } | null = null - constructor(private readonly ceilingMs = STARTUP_GATE_CEILING_MS) {} + constructor( + private readonly ceilingMs = STARTUP_GATE_CEILING_MS, + private readonly now: () => number = Date.now + ) {} /** Closes the gate until it is opened; a no-op while it is closed. Before any request is served. */ hold(): void { if (this.pending) { return } + this.heldAt = this.now() this.pending = new Promise((resolve) => { this.release = resolve }) @@ -28,30 +45,65 @@ export class StructuredAgentSessionStartupGate { console.warn( '[structured-agent-session] startup settle still running; chat commands go ahead' ) - this.open() + this.open('ceiling') }, this.ceilingMs) this.ceiling.unref?.() } + /** The startup step began; only timed. */ + stepStarted(): void { + this.step ??= { startedAt: this.now(), endedAt: null } + } + /** Opens once the settle ends, whether it settled or failed. */ openWhen(settled: Promise): void { - void settled - .catch((error: unknown) => { + void settled.then( + () => this.endStep('settle ended'), + (error: unknown) => { console.warn('[structured-agent-session] startup settle failed', error) - }) - .finally(() => this.open()) + this.endStep('settle failed') + } + ) } - open(): void { + /** Opens now: the step ended with nothing to settle, or failed before its settle began. */ + open(by: StartupGateOpener = 'nothing to settle'): void { if (this.ceiling) { clearTimeout(this.ceiling) this.ceiling = null } + if (this.pending && !this.opened) { + this.opened = { at: this.now(), by } + if ((by === 'nothing to settle' || by === 'step failed') && this.step?.endedAt === null) { + this.step.endedAt = this.opened.at + } + } this.release?.() this.release = null this.pending = null + this.report() } /** What a chat command waits for, or null when the gate is open. */ ready = (): Promise | null => this.pending + + private endStep(by: StartupGateOpener): void { + if (this.step && this.step.endedAt === null) { + this.step.endedAt = this.now() + } + this.open(by) + } + + private report(): void { + const { heldAt, step, opened } = this + if (heldAt === null || !opened || (step && step.endedAt === null)) { + return + } + const since = (at: number | null | undefined) => (at == null ? 'never' : `+${at - heldAt} ms`) + console.info( + `[structured-agent-session] startup gate: held at launch; step started ${since(step?.startedAt)}, ` + + `ended ${since(step?.endedAt)}; opened ${since(opened.at)} by ${opened.by}` + ) + this.heldAt = null + } } diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index fa7d546c3ca4..aab947192e9a 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -18,7 +18,7 @@ import { type RestTestRig } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' import { OrcaRuntimeService } from './orca-runtime' -import type { StructuredAgentSessionStartupGate } from './structured-agent-session-startup-gate' +import { StructuredAgentSessionStartupGate } from './structured-agent-session-startup-gate' type PrepareInternals = { structuredAgentSessionStartupGate: StructuredAgentSessionStartupGate @@ -75,6 +75,7 @@ it('holds chat commands until the startup settle ends, and then lets them throug await rig.boot() const runtime = restartedRuntime() const gate = internals(runtime).structuredAgentSessionStartupGate + const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) runtime.holdStructuredAgentSessionCommandsForStartup() expect(gate.ready()).not.toBeNull() @@ -82,6 +83,9 @@ it('holds chat commands until the startup settle ends, and then lets them throug await vi.waitFor(() => expect(owes('session-crashed')).toBe(false)) await vi.waitFor(() => expect(gate.ready()).toBeNull()) + expect(timing).toHaveBeenCalledWith( + expect.stringMatching(/step started \+\d+ ms, ended \+\d+ ms; opened \+\d+ ms by settle ended$/) + ) }) it('opens the gate when the startup step fails, never stranding a command', async () => { @@ -95,9 +99,30 @@ it('opens the gate when the startup step fails, never stranding a command', asyn vi.spyOn(console, 'warn').mockImplementation(() => undefined) runtime.holdStructuredAgentSessionCommandsForStartup() + const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) expect(gate.ready()).toBeNull() + expect(timing).toHaveBeenCalledWith(expect.stringMatching(/by step failed$/)) +}) + +it('logs one timing line per launch, naming the ceiling when it opened the gate first', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) + const gate = new StructuredAgentSessionStartupGate(5) + gate.hold() + gate.stepStarted() + let endSettle = () => {} + gate.openWhen(new Promise((resolve) => (endSettle = resolve))) + await vi.waitFor(() => expect(gate.ready()).toBeNull()) + // Written once the step is over too, so it carries both times. + expect(timing).not.toHaveBeenCalled() + + endSettle() + await vi.waitFor(() => expect(timing).toHaveBeenCalledOnce()) + expect(timing).toHaveBeenCalledWith( + expect.stringMatching(/ended \+\d+ ms; opened \+\d+ ms by ceiling$/) + ) }) it('seeds and settles on a host no client ever lists (T8)', async () => { diff --git a/src/main/startup/main-process-runtime-launch-activation.test.ts b/src/main/startup/main-process-runtime-launch-activation.test.ts index be1ebdac440a..5298ec5feca2 100644 --- a/src/main/startup/main-process-runtime-launch-activation.test.ts +++ b/src/main/startup/main-process-runtime-launch-activation.test.ts @@ -153,9 +153,10 @@ describe('desktop startup activation', () => { launchHooks.failBeforeWindow = false state.mainWindow = null state.isServeMode = false - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime and holds chat commands before the mocked RPC server takes it. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime, holds chat commands and starts the chat startup step before the mocked RPC server takes it. state.runtime = { - holdStructuredAgentSessionCommandsForStartup: () => undefined + holdStructuredAgentSessionCommandsForStartup: () => undefined, + startStructuredAgentSessionStartupAfter: () => undefined } as unknown as NonNullable // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only calls whenReady(). state.windowsShellPathHydration = { diff --git a/src/main/startup/main-process-runtime-launch.ts b/src/main/startup/main-process-runtime-launch.ts index 3af2e34c435a..808ca51d9b08 100644 --- a/src/main/startup/main-process-runtime-launch.ts +++ b/src/main/startup/main-process-runtime-launch.ts @@ -315,6 +315,10 @@ export async function initializeMainProcessRuntimeLaunch( // Why published: the renderer's git-environment barrier must fence on the same // generation the terminal startup services wait for, not a later re-read. state.shellPathReady = shellPathReady + // The host seeds its chats' statuses and settles crashed ones itself, so a window, a remote + // client or neither sees the same state. It needs neither the terminal daemon nor the hook + // server, so it does not wait for the first window's services. + runtime.startStructuredAgentSessionStartupAfter(shellPathReady) // Why before any window: the poisoned install DACL kills the renderer at init, and // the probe that detects it cannot finish before createMainWindow. Bounded, and a // no-op (one absent-file read) unless a previous launch already recorded the verdict. @@ -339,10 +343,8 @@ export async function initializeMainProcessRuntimeLaunch( await shellPathReady bindTerminalRuntimeStartupServices(Promise.resolve(startTerminalRuntimeStartupServices())) } - // The host seeds its chats' statuses and settles crashed ones itself, so a window, a remote - // client or neither sees the same state. Only behind the first-window services: the PTY - // inventory it takes before the lease check needs the daemon provider. The managed-WSL barrier - // gates terminal spawns, which this step makes none of. + // The tab restore's terminal records refresh lists the daemon's terminals. The first-window + // services resolve when the daemon and hook server are up, or at their 12 s timeout regardless. void state.firstWindowStartupServicesReady .then(() => runtime.prepareStructuredAgentSessionStartupRestoration()) .catch((error: unknown) => console.warn('[structured-agent-session] startup failed', error)) From 3ebcd7aedabbd7e4f7ed50f5df8868e05cced066 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:58:33 -0700 Subject: [PATCH 042/107] fix(native-chat): the startup timing line records the step whoever opened the wait --- ...structured-agent-session-create-support.ts | 12 ++- .../structured-agent-session-startup-gate.ts | 63 +++++++------ ...-agent-session-startup-restoration.test.ts | 93 +++++++++++++++---- 3 files changed, 115 insertions(+), 53 deletions(-) diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index 63dfbef92885..96107b457bf8 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -25,6 +25,7 @@ import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' import { parseWslUncPath } from '../../shared/wsl-paths' import { parseWorkspaceKey } from '../../shared/workspace-scope' import { applyStructuredCodexWorkspaceTrust } from '../agent-workspace-trust-spawn' +import type { StartupStepOutcome } from './structured-agent-session-startup-gate' export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaRuntimeWithGetWorktreePs { // The history restore a tab restore owes, until a caller that answered with its list starts it. @@ -310,11 +311,12 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR const gate = this.structuredAgentSessionStartupGate gate.stepStarted() // Chat commands held for startup go ahead however this ends: when the settle does, or now. - let opener: 'nothing to settle' | 'step failed' | null = 'nothing to settle' + let ended: StartupStepOutcome | null = 'no chats on disk' try { if (!this.hasPersistedStructuredAgentSessionStore()) { return } + ended = 'no host' // A refused host is no host: startup goes on, and only structured requests are refused. await ensureStructuredAgentSessionHostUnlessRefused(() => this.ensureStructuredAgentSessionHost() @@ -326,17 +328,17 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR host, this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null, (settled) => { - opener = null + ended = null gate.openWhen(settled) } ) } } catch (error) { - opener &&= 'step failed' + ended &&= 'step failed' throw error } finally { - if (opener) { - gate.open(opener) + if (ended) { + gate.stepEnded(ended) } } } diff --git a/src/main/runtime/structured-agent-session-startup-gate.ts b/src/main/runtime/structured-agent-session-startup-gate.ts index fbba5d3f6e9a..ddf65deae636 100644 --- a/src/main/runtime/structured-agent-session-startup-gate.ts +++ b/src/main/runtime/structured-agent-session-startup-gate.ts @@ -10,21 +10,28 @@ // safe. Past it, the order is what is lost: a crashed turn may get its verdict from an open. const STARTUP_GATE_CEILING_MS = 10_000 -type StartupGateOpener = +/** How the startup step ended. */ +export type StartupStepOutcome = | 'settle ended' | 'settle failed' | 'step failed' - | 'ceiling' - | 'nothing to settle' + | 'no chats on disk' + | 'no host' + +type StartupGateOpener = StartupStepOutcome | 'ceiling' export class StructuredAgentSessionStartupGate { private pending: Promise | null = null private release: (() => void) | null = null private ceiling: ReturnType | null = null - // One timing line per launch, for the startup measurements: written once the gate is open and - // the step is over (or never started). + // One timing line per launch, for the startup measurements. The step's times are recorded + // whenever they happen, whoever opened the gate; the line is written once the gate is open and + // the step is over. private heldAt: number | null = null - private step: { startedAt: number; endedAt: number | null } | null = null + private step: { + startedAt: number + ended: { at: number; outcome: StartupStepOutcome } | null + } | null = null private opened: { at: number; by: StartupGateOpener } | null = null constructor( @@ -52,31 +59,38 @@ export class StructuredAgentSessionStartupGate { /** The startup step began; only timed. */ stepStarted(): void { - this.step ??= { startedAt: this.now(), endedAt: null } + this.step ??= { startedAt: this.now(), ended: null } + } + + /** The startup step is over: timed, and the gate opens now unless the ceiling opened it. */ + stepEnded(outcome: StartupStepOutcome): void { + const at = this.now() + this.step ??= { startedAt: at, ended: null } + this.step.ended ??= { at, outcome } + this.open(outcome) } - /** Opens once the settle ends, whether it settled or failed. */ + /** The step ends when the settle does, whether it settled or failed. */ openWhen(settled: Promise): void { void settled.then( - () => this.endStep('settle ended'), + () => this.stepEnded('settle ended'), (error: unknown) => { console.warn('[structured-agent-session] startup settle failed', error) - this.endStep('settle failed') + this.stepEnded('settle failed') } ) } - /** Opens now: the step ended with nothing to settle, or failed before its settle began. */ - open(by: StartupGateOpener = 'nothing to settle'): void { + /** What a chat command waits for, or null when the gate is open. */ + ready = (): Promise | null => this.pending + + private open(by: StartupGateOpener): void { if (this.ceiling) { clearTimeout(this.ceiling) this.ceiling = null } - if (this.pending && !this.opened) { + if (this.pending) { this.opened = { at: this.now(), by } - if ((by === 'nothing to settle' || by === 'step failed') && this.step?.endedAt === null) { - this.step.endedAt = this.opened.at - } } this.release?.() this.release = null @@ -84,25 +98,14 @@ export class StructuredAgentSessionStartupGate { this.report() } - /** What a chat command waits for, or null when the gate is open. */ - ready = (): Promise | null => this.pending - - private endStep(by: StartupGateOpener): void { - if (this.step && this.step.endedAt === null) { - this.step.endedAt = this.now() - } - this.open(by) - } - private report(): void { const { heldAt, step, opened } = this - if (heldAt === null || !opened || (step && step.endedAt === null)) { + if (heldAt === null || !opened || !step?.ended) { return } - const since = (at: number | null | undefined) => (at == null ? 'never' : `+${at - heldAt} ms`) console.info( - `[structured-agent-session] startup gate: held at launch; step started ${since(step?.startedAt)}, ` + - `ended ${since(step?.endedAt)}; opened ${since(opened.at)} by ${opened.by}` + `[structured-agent-session] startup gate: held at launch; step started +${step.startedAt - heldAt} ms, ` + + `ended +${step.ended.at - heldAt} ms (${step.ended.outcome}); opened +${opened.at - heldAt} ms by ${opened.by}` ) this.heldAt = null } diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index aab947192e9a..e15759a6836d 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -2,7 +2,7 @@ // headless host has its seeded statuses and its settled crashed chats, and no failure in the step // (one chat's open, one chat's settlement) costs startup or the other chats. -import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { closeTestJournalHostDatabases, readTestJournalSessionStatus @@ -84,7 +84,9 @@ it('holds chat commands until the startup settle ends, and then lets them throug await vi.waitFor(() => expect(gate.ready()).toBeNull()) expect(timing).toHaveBeenCalledWith( - expect.stringMatching(/step started \+\d+ ms, ended \+\d+ ms; opened \+\d+ ms by settle ended$/) + expect.stringMatching( + /step started \+\d+ ms, ended \+\d+ ms \(settle ended\); opened \+\d+ ms by settle ended$/ + ) ) }) @@ -103,26 +105,81 @@ it('opens the gate when the startup step fails, never stranding a command', asyn await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) expect(gate.ready()).toBeNull() - expect(timing).toHaveBeenCalledWith(expect.stringMatching(/by step failed$/)) + expect(timing).toHaveBeenCalledOnce() + expect(timing).toHaveBeenCalledWith( + expect.stringMatching(/\(step failed\); opened \+\d+ ms by step failed$/) + ) }) -it('logs one timing line per launch, naming the ceiling when it opened the gate first', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) - const gate = new StructuredAgentSessionStartupGate(5) - gate.hold() - gate.stepStarted() - let endSettle = () => {} - gate.openWhen(new Promise((resolve) => (endSettle = resolve))) - await vi.waitFor(() => expect(gate.ready()).toBeNull()) - // Written once the step is over too, so it carries both times. - expect(timing).not.toHaveBeenCalled() +describe('one timing line per launch, with the real step times whoever opened the gate', () => { + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + const CEILING_MS = 5 + let timing: ReturnType - endSettle() - await vi.waitFor(() => expect(timing).toHaveBeenCalledOnce()) - expect(timing).toHaveBeenCalledWith( - expect.stringMatching(/ended \+\d+ ms; opened \+\d+ ms by ceiling$/) + beforeEach(() => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) + }) + + /** Exactly one line, for a gate the ceiling opened before the step ended `outcome`. */ + async function expectOneCeilingLine(outcome: string, started = /\+\d+ ms/): Promise { + await vi.waitFor(() => expect(timing).toHaveBeenCalledOnce()) + await sleep(20) + expect(timing).toHaveBeenCalledOnce() + const line = String(timing.mock.calls[0]?.[0]) + expect(line).toMatch( + new RegExp(`ended \\+\\d+ ms \\(${outcome}\\); opened \\+\\d+ ms by ceiling$`) + ) + expect(line).toMatch(new RegExp(`step started ${started.source},`)) + } + + it.each(['settle ended', 'no host', 'step failed'] as const)( + 'writes it when the step ends after the ceiling: %s', + async (outcome) => { + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + gate.stepStarted() + await vi.waitFor(() => expect(gate.ready()).toBeNull()) + expect(timing).not.toHaveBeenCalled() + + gate.stepEnded(outcome) + + await expectOneCeilingLine(outcome) + } ) + + it('times a step that starts only after the ceiling', async () => { + const gate = new StructuredAgentSessionStartupGate(CEILING_MS) + gate.hold() + await vi.waitFor(() => expect(gate.ready()).toBeNull()) + await sleep(10) + expect(timing).not.toHaveBeenCalled() + + gate.stepStarted() + gate.openWhen(Promise.resolve()) + + await expectOneCeilingLine('settle ended', /\+([5-9]|\d{2,}) ms/) + }) + + it.each([ + ['throws', 'step failed'], + ['builds no host', 'no host'] + ] as const)('writes it when a slow host build past the ceiling %s', async (how, outcome) => { + const runtime = restartedRuntime() + const internal = internals(runtime) + internal.structuredAgentSessionStartupGate = new StructuredAgentSessionStartupGate(CEILING_MS) + internal.ensureStructuredAgentSessionHost = async () => { + await sleep(30) + if (how === 'throws') { + throw new Error('host build failed') + } + } + runtime.holdStructuredAgentSessionCommandsForStartup() + + await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) + + await expectOneCeilingLine(outcome) + }) }) it('seeds and settles on a host no client ever lists (T8)', async () => { From 72def96a0c29e4be564f5b5273710ca8eb8ad4a3 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:06:10 -0700 Subject: [PATCH 043/107] fix(native-chat): the startup gate closes once per launch --- .../structured-agent-session-startup-gate.ts | 7 +++++-- ...tured-agent-session-startup-restoration.test.ts | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/runtime/structured-agent-session-startup-gate.ts b/src/main/runtime/structured-agent-session-startup-gate.ts index ddf65deae636..eac65121f775 100644 --- a/src/main/runtime/structured-agent-session-startup-gate.ts +++ b/src/main/runtime/structured-agent-session-startup-gate.ts @@ -24,6 +24,7 @@ export class StructuredAgentSessionStartupGate { private pending: Promise | null = null private release: (() => void) | null = null private ceiling: ReturnType | null = null + private held = false // One timing line per launch, for the startup measurements. The step's times are recorded // whenever they happen, whoever opened the gate; the line is written once the gate is open and // the step is over. @@ -39,11 +40,13 @@ export class StructuredAgentSessionStartupGate { private readonly now: () => number = Date.now ) {} - /** Closes the gate until it is opened; a no-op while it is closed. Before any request is served. */ + /** Closes the gate until it is opened, once per launch, before any request is served. */ hold(): void { - if (this.pending) { + // One-shot: re-closing after it opened could hold /clear's replacement attach under a chat lock. + if (this.held) { return } + this.held = true this.heldAt = this.now() this.pending = new Promise((resolve) => { this.release = resolve diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index e15759a6836d..65810cf8cd7d 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -111,6 +111,20 @@ it('opens the gate when the startup step fails, never stranding a command', asyn ) }) +it('closes the gate once per launch: a hold after it opened leaves it open', async () => { + const gate = new StructuredAgentSessionStartupGate() + gate.hold() + gate.hold() + vi.spyOn(console, 'info').mockImplementation(() => undefined) + gate.stepStarted() + gate.stepEnded('no chats on disk') + expect(gate.ready()).toBeNull() + + gate.hold() + + expect(gate.ready()).toBeNull() +}) + describe('one timing line per launch, with the real step times whoever opened the gate', () => { const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) const CEILING_MS = 5 From 4f266e8e293d670acd76a6439a0a584317278122 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:37:55 -0700 Subject: [PATCH 044/107] fix(native-chat): the startup wait lets go before the AI vault's 5 s chat restore gives up --- .../structured-agent-session-startup-gate.ts | 10 ++++++---- ...ctured-agent-session-startup-restoration.test.ts | 13 +++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/runtime/structured-agent-session-startup-gate.ts b/src/main/runtime/structured-agent-session-startup-gate.ts index eac65121f775..f35b526eab0b 100644 --- a/src/main/runtime/structured-agent-session-startup-gate.ts +++ b/src/main/runtime/structured-agent-session-startup-gate.ts @@ -5,10 +5,12 @@ // those locks); none is refused. It always opens: when the settle ends, however it ends, or at a // ceiling, so a startup fault never strands a command. -// Under the shortest client request timeout (15 s), counted from launch (`hold`): a held command -// lets go before its caller gives up, and an open settles its own chat, so letting go early is -// safe. Past it, the order is what is lost: a crashed turn may get its verdict from an open. -const STARTUP_GATE_CEILING_MS = 10_000 +// Under the shortest client timeout on a held call, with margin: the AI vault's chat restore gives +// its reveal 5 s (activate-ai-vault-structured-session.ts); remote desktop and phone give 15 s. +// Counted from launch (`hold`). A held command lets go before its caller gives up, and an open +// settles its own chat, so letting go early is safe. Past it, the order is what is lost: a crashed +// turn may get its verdict from an open. +const STARTUP_GATE_CEILING_MS = 4_000 /** How the startup step ended. */ export type StartupStepOutcome = diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index 65810cf8cd7d..49d92e4db803 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -111,6 +111,19 @@ it('opens the gate when the startup step fails, never stranding a command', asyn ) }) +it('lets held commands go before the shortest client timeout on a held call (the AI vault restore, 5 s)', () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.useFakeTimers() + try { + const gate = new StructuredAgentSessionStartupGate() + gate.hold() + vi.advanceTimersByTime(4_500) + expect(gate.ready()).toBeNull() + } finally { + vi.useRealTimers() + } +}) + it('closes the gate once per launch: a hold after it opened leaves it open', async () => { const gate = new StructuredAgentSessionStartupGate() gate.hold() From 9dd385260747b8a536f7217bd37991f4d4695006 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:49:42 -0700 Subject: [PATCH 045/107] fix(native-chat): a startup recovery that never answers leaves its lease unverified and the settle goes on --- ...ctured-agent-session-startup-state.test.ts | 36 +++++++++++++++++++ .../structured-agent-session-startup-state.ts | 19 +++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 0f63a0170abb..8767dc166039 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -589,6 +589,42 @@ function conversationCommandParams(sessionId: string, command: 'clear') { } } +describe('a recovery that never answers', () => { + it('leaves that lease unverified and lets the settle go on', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const stuck = { ...agentSessionRecordFixture(), sessionId: 'session-stuck' } + stuck.lease = { ...stuck.lease, sessionId: 'session-stuck', handoffStage: 'recovering' } + const resolveRecovery = vi.fn(() => new Promise(() => {})) + const restoreListed = vi.fn(async () => undefined) + const openDeps = { + store: { getRecord: () => stuck, listRecords: () => [stuck] }, + journalDatabase: { readOnly: false, db: { prepare: () => ({ all: () => [] }) } } + } + const state = createStructuredAgentSessionStartupState({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the settle reads only the store's records and the unsettled-row query; nothing is selected, so nothing opens. + openDeps: openDeps as unknown as StructuredAgentSessionStartupStateDeps['openDeps'], + canSettle: (record): record is AgentSessionRecord => record !== null, + seedStatus: vi.fn(), + resolveRecovery, + restoreListed, + serialize: (_sessionId, task) => task(), + hasSession: () => false, + isDisposed: () => false, + recoveryBudgetMs: 20 + }) + + await state.settleOwedSessions([]) + + expect(resolveRecovery).toHaveBeenCalledWith('session-stuck') + expect(restoreListed).toHaveBeenCalledWith([]) + expect(stuck.lease.handoffStage).toBe('recovering') + expect(warn).toHaveBeenCalledWith( + '[structured-agent-session] a chat recovery outlasted startup; left unverified', + { sessionId: 'session-stuck' } + ) + }) +}) + describe('a stored status no settle here can clear (R2A-4)', () => { it('is refused by the shared closed-chat settle, which opens nothing for it', async () => { const opens: string[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 75169ebabc80..0ad9b012c24e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -14,6 +14,7 @@ import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' +import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { @@ -30,6 +31,10 @@ import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-rea // Record-only (a probe and at most a process stop each), so a few at once. const RECOVERY_CONCURRENCY = 4 +// A recovery is process probes around at most one stop (its SIGTERM grace is 5.5 s). Past this a +// probe has stopped answering: the lease stays `recovering`, unverified and never called dead, for +// the next attach or send to resolve, and the settle goes on. +const RECOVERY_BUDGET_MS = 15_000 export type StructuredAgentSessionStartupStateDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { @@ -46,6 +51,8 @@ export type StructuredAgentSessionStartupStateDeps = { resolveRecovery: (sessionId: string) => Promise /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish). */ restoreListed: (records: AgentSessionRecord[]) => Promise + /** Tests shorten it; production takes the default. */ + recoveryBudgetMs?: number serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean isDisposed: () => boolean @@ -212,8 +219,18 @@ async function resolveRecoveringLeases(deps: StructuredAgentSessionStartupStateD const recovering = deps.openDeps.store .listRecords() .filter((record) => record.lease.handoffStage === 'recovering') + const budgetMs = deps.recoveryBudgetMs ?? RECOVERY_BUDGET_MS await forEachWithConcurrency(recovering, RECOVERY_CONCURRENCY, async ({ sessionId }) => { - await deps.resolveRecovery(sessionId) + if ( + (await withTimeout(deps.resolveRecovery(sessionId), budgetMs, null)) === null + ) { + console.warn( + '[structured-agent-session] a chat recovery outlasted startup; left unverified', + { + sessionId + } + ) + } }) } From 5735ee94e1904fc1f814ac9c311741ef0bc115ed Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:16:22 -0700 Subject: [PATCH 046/107] chore: put back main's orcad files the merge commit's formatter rewrote The pre-commit formatter ran over every file the merge staged and reformatted three of main's orcad files that this PR does not touch. They go back to main's exact content; the hook is skipped for this commit only, because it would rewrite them again. --- .../ipc/orcad-runtime-maintenance-handlers.ts | 5 +- src/main/ssh/orcad-runtime-decommission.ts | 5 +- src/main/ssh/orcad-runtime-maintenance.ts | 228 +++++++++--------- 3 files changed, 110 insertions(+), 128 deletions(-) diff --git a/src/main/ipc/orcad-runtime-maintenance-handlers.ts b/src/main/ipc/orcad-runtime-maintenance-handlers.ts index 0f16da4de074..1fe7c49cdda8 100644 --- a/src/main/ipc/orcad-runtime-maintenance-handlers.ts +++ b/src/main/ipc/orcad-runtime-maintenance-handlers.ts @@ -23,10 +23,7 @@ export function registerOrcadRuntimeMaintenanceHandlers(options: { }): void { ipcMain.handle( 'runtimeEnvironments:updateOrcad', - async ( - _event, - args: { selector: string; force?: boolean } - ): Promise => { + async (_event, args: { selector: string; force?: boolean }): Promise => { const result = await updateManagedOrcadEnvironment(options.getUserDataPath(), { selector: requiredString(args?.selector, 'Server'), force: args?.force === true diff --git a/src/main/ssh/orcad-runtime-decommission.ts b/src/main/ssh/orcad-runtime-decommission.ts index 4eb9f335fa0a..ffb8d36954cf 100644 --- a/src/main/ssh/orcad-runtime-decommission.ts +++ b/src/main/ssh/orcad-runtime-decommission.ts @@ -7,10 +7,7 @@ import type { OrcadManagedStopResult } from '../../shared/orcad-managed-runtime' import { removeManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' -import type { - KnownRuntimeEnvironment, - OrcadDeploymentLink -} from '../../shared/runtime-environments' +import type { KnownRuntimeEnvironment, OrcadDeploymentLink } from '../../shared/runtime-environments' import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' import { withStaleOrcadActivationRecoveryLock } from './orcad-activation-lock' import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' diff --git a/src/main/ssh/orcad-runtime-maintenance.ts b/src/main/ssh/orcad-runtime-maintenance.ts index 712d0d67ae00..5ed3a8e16f52 100644 --- a/src/main/ssh/orcad-runtime-maintenance.ts +++ b/src/main/ssh/orcad-runtime-maintenance.ts @@ -72,114 +72,106 @@ export function updateManagedOrcadEnvironment( userDataPath: string, args: LifecycleArgs & { force?: boolean } ): Promise { - return withManagedOrcadLifecycle( - userDataPath, - args.selector, - async ({ environment, deployment }) => { - const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) - const census = await collectManagedTerminalCensus( - userDataPath, - environment, - context.activationRecord - ) - const localOrcadDir = await materializeOrcadArtifact(context.serverTarget, { - signal: args.signal - }) - const result = await deployOrcad({ - ...managedOrcadSlot(context, deployment.remotePort, args.signal), - localOrcadDir, - target: context.serverTarget, - census, - force: args.force - }) - if (result.outcome === 'installed-not-activated') { - const deferral = { - outcome: 'deferred' as const, - candidateVersion: result.fullVersion, - code: result.code, - reason: result.reason, - forceable: isForceableOrcadDeferral(result.code) - } - recordManagedOrcadUpdateDeferral(environment.id, deferral) - return deferral - } - clearManagedOrcadUpdateDeferral(environment.id) - const readiness = await probeManagedOrcadReadiness( - context, - localOrcadDir, - result.fullVersion, - args.signal - ) - const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) - return { - outcome: result.outcome === 'already-active' ? 'already-current' : 'updated', - environment: redactRuntimeEnvironment(updated), - activeVersion: result.fullVersion + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const census = await collectManagedTerminalCensus( + userDataPath, + environment, + context.activationRecord + ) + const localOrcadDir = await materializeOrcadArtifact(context.serverTarget, { + signal: args.signal + }) + const result = await deployOrcad({ + ...managedOrcadSlot(context, deployment.remotePort, args.signal), + localOrcadDir, + target: context.serverTarget, + census, + force: args.force + }) + if (result.outcome === 'installed-not-activated') { + const deferral = { + outcome: 'deferred' as const, + candidateVersion: result.fullVersion, + code: result.code, + reason: result.reason, + forceable: isForceableOrcadDeferral(result.code) } + recordManagedOrcadUpdateDeferral(environment.id, deferral) + return deferral } - ) + clearManagedOrcadUpdateDeferral(environment.id) + const readiness = await probeManagedOrcadReadiness( + context, + localOrcadDir, + result.fullVersion, + args.signal + ) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: result.outcome === 'already-active' ? 'already-current' : 'updated', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.fullVersion + } + }) } export function rollbackManagedOrcadEnvironment( userDataPath: string, args: LifecycleArgs ): Promise { - return withManagedOrcadLifecycle( - userDataPath, - args.selector, - async ({ environment, deployment }) => { - const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) - const record = context.activationRecord - const target = record.previous - if (!target) { - return { - outcome: 'refused', - code: 'orcad_rollback_no_target', - reason: 'This server has no previous version to roll back to.' - } - } - const census = await collectManagedTerminalCensus(userDataPath, environment, record) - // Why idle only: this client cannot read the older build's daemon protocol, so it cannot show - // that build would reach terminals that are still running. - if (census.liveSessions !== 0) { - return { - outcome: 'refused', - code: - census.liveSessions === null - ? 'orcad_rollback_census_unavailable' - : 'orcad_rollback_terminals_running', - reason: - census.liveSessions === null - ? 'The server did not answer how many terminals it runs. Retry when it answers.' - : 'Close the terminals running on this server before rolling it back.' - } - } - const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) - const targetDir = managedOrcadInstallDir(context, target) - const targetBuildHash = await readRemoteOrcadBuildHash(slot, targetDir) - const result = await rollbackOrcad({ - ...slot, - record, - census, - targetBuildHash, - targetDaemonProtocol: CURRENT_ORCAD_DAEMON_PROTOCOL - }) - if (result.outcome !== 'rolled-back') { - return result + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const record = context.activationRecord + const target = record.previous + if (!target) { + return { + outcome: 'refused', + code: 'orcad_rollback_no_target', + reason: 'This server has no previous version to roll back to.' } - const readiness = await probeActiveOrcadReadiness( - { ...slot, remoteInstallDir: targetDir }, - { buildHash: targetBuildHash, fullVersion: result.target } - ) - const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + } + const census = await collectManagedTerminalCensus(userDataPath, environment, record) + // Why idle only: this client cannot read the older build's daemon protocol, so it cannot show + // that build would reach terminals that are still running. + if (census.liveSessions !== 0) { return { - outcome: 'rolled-back', - environment: redactRuntimeEnvironment(updated), - activeVersion: result.target, - discarded: result.discarded + outcome: 'refused', + code: + census.liveSessions === null + ? 'orcad_rollback_census_unavailable' + : 'orcad_rollback_terminals_running', + reason: + census.liveSessions === null + ? 'The server did not answer how many terminals it runs. Retry when it answers.' + : 'Close the terminals running on this server before rolling it back.' } } - ) + const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) + const targetDir = managedOrcadInstallDir(context, target) + const targetBuildHash = await readRemoteOrcadBuildHash(slot, targetDir) + const result = await rollbackOrcad({ + ...slot, + record, + census, + targetBuildHash, + targetDaemonProtocol: CURRENT_ORCAD_DAEMON_PROTOCOL + }) + if (result.outcome !== 'rolled-back') { + return result + } + const readiness = await probeActiveOrcadReadiness( + { ...slot, remoteInstallDir: targetDir }, + { buildHash: targetBuildHash, fullVersion: result.target } + ) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: 'rolled-back', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.target, + discarded: result.discarded + } + }) } /** Finishes or undoes an interrupted activation, rollback or decommission on the host. */ @@ -187,29 +179,25 @@ export function recoverManagedOrcadEnvironment( userDataPath: string, args: LifecycleArgs ): Promise { - return withManagedOrcadLifecycle( - userDataPath, - args.selector, - async ({ environment, deployment }) => { - const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) - const result = await recoverInterruptedOrcadActivation( - managedOrcadSlot(context, deployment.remotePort, args.signal) - ) - if (result.outcome !== 'recovered') { - return result - } - const updated = result.readiness - ? refreshPairing(userDataPath, environment, result.readiness, deployment.localPort) - : environment - if (result.activeVersion) { - await ensureOrcadManagedTunnel(userDataPath, environment.id) - } - return { - outcome: 'recovered', - resolution: result.resolution, - activeVersion: result.activeVersion, - environment: redactRuntimeEnvironment(updated) - } + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const result = await recoverInterruptedOrcadActivation( + managedOrcadSlot(context, deployment.remotePort, args.signal) + ) + if (result.outcome !== 'recovered') { + return result } - ) + const updated = result.readiness + ? refreshPairing(userDataPath, environment, result.readiness, deployment.localPort) + : environment + if (result.activeVersion) { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + } + return { + outcome: 'recovered', + resolution: result.resolution, + activeVersion: result.activeVersion, + environment: redactRuntimeEnvironment(updated) + } + }) } From b122e38112571076c0ff506b3c283208101330f2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:31:07 -0700 Subject: [PATCH 047/107] test(native-chat): say why the status feed's stand-in journals are cast --- src/main/agent-hooks/first-work-branch-rename.test.ts | 2 ++ .../rpc/methods/structured-agent-session-rpc.test-fixture.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index de9d2c6181bc..2421ba440816 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -105,6 +105,7 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { // A real journal's sequence only ever advances, so the feed's projection // cache must miss on every publish here: this test is about the rename. let sequence = 0 + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (snapshot, cursor, its projection, activity, read-only). const journal = { snapshot: () => ({ items }), lastActivityAt: () => 1, @@ -214,6 +215,7 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { } } ] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (snapshot, cursor, its projection, activity, read-only). const journal = { isReadOnly: false, lastActivityAt: () => 1, diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index 2f14fdf9b051..46cb1e8c4d22 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -102,6 +102,7 @@ function statusFeed(): StructuredAgentSessionStatusFeed { [ STATUS_SESSION, { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (snapshot, cursor, its projection, activity, read-only). journal: { isReadOnly: false, cursor: () => ({ epoch: 'epoch-status', sequence: 2 }), From 56291e6d405b78d2b09340d19d472cd8b501fd61 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:34:13 -0700 Subject: [PATCH 048/107] fix(native-chat): the startup step's memo lives with the step; a stand-in journal offers its projection --- .../claude-background-wake-retention.test.ts | 21 ++++++++++++------- ...structured-agent-session-create-support.ts | 1 + src/main/runtime/orca-runtime-runtime-id.ts | 1 - 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/claude/claude-background-wake-retention.test.ts b/src/main/claude/claude-background-wake-retention.test.ts index a865e1c1b76e..b6d4c53daa1b 100644 --- a/src/main/claude/claude-background-wake-retention.test.ts +++ b/src/main/claude/claude-background-wake-retention.test.ts @@ -7,6 +7,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { AgentJournalSubmission } from '../../shared/agent-session-journal-types' import { structuredRunningChildWork } from '../../shared/agent-child-work-listing' +import { projectStructuredAgentSessionStatusState } from '../../shared/structured-agent-session-projection' import type { AgentHookServer } from '../agent-hooks/server' import type { StructuredAgentSessionJournalProjections } from '../native-chat/agent-session-wire/structured-agent-session-status-journal-projection' import { StructuredAgentSessionStatusFeed } from '../native-chat/agent-session-wire/structured-agent-session-status-feed' @@ -29,16 +30,22 @@ async function wiredSession() { const run = await producer(host) const submissions: AgentJournalSubmission[] = [] let sequence = 0 + const snapshot = () => ({ + items: [...run.journalItems.values()] + .sort((a, b) => a.sequence - b.sequence) + .map((item) => ({ ...item, revision: 1, observedAt: item.sequence })), + submissions: submissions.map((submission) => ({ ...submission })) + }) const journal = { cursor: () => ({ epoch: 'epoch-1', sequence: ++sequence }), isReadOnly: false, lastActivityAt: () => 1, - snapshot: () => ({ - items: [...run.journalItems.values()] - .sort((a, b) => a.sequence - b.sequence) - .map((item) => ({ ...item, revision: 1 })), - submissions: submissions.map((submission) => ({ ...submission })) - }) + snapshot, + // The journal's own projection, which the feed shares with the status stored beside it. + statusState: (fence?: number) => { + const { items, submissions: sent } = snapshot() + return projectStructuredAgentSessionStatusState(items, sent, fence) + } } const feed = new StructuredAgentSessionStatusFeed({ logger: createStructuredAgentSessionLogger(), @@ -46,7 +53,7 @@ async function wiredSession() { [ parent.sessionId, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the feed reads only the cursor, read-only flag, activity clock and snapshot served here. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the feed reads only the cursor, read-only flag, activity clock, snapshot and projection served here. journal: journal as unknown as Journal, params: { location: { diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index 96107b457bf8..0505c1e67f81 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -32,6 +32,7 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR protected owedStructuredAgentSessionHistoryRestore: (() => void) | null = null // Listed chats startup could not answer from stored state; null until it has run. protected structuredAgentSessionBackgroundRestoreIds: string[] | null = null + protected structuredAgentSessionStartupStepPromise: Promise | null = null async getStructuredAgentSessionCreateSupport( worktreeSelector: string, diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index 090e6665524c..c85b4a51deb5 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -129,7 +129,6 @@ export class OrcaRuntimeWithRuntimeId { protected structuredAgentSessionInventoryUnverifiable = false protected structuredAgentSessionStartupRestorePromise: Promise | null = null - protected structuredAgentSessionStartupStepPromise: Promise | null = null protected mobileSessionTabsChangeSequence = 0 From bc668923a3663976bff1717552a2f543b51761f9 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:40:03 -0700 Subject: [PATCH 049/107] test(native-chat): a failed Stop or Resume row leaves the queue's pause as a replay reads it --- .../agent-session-journal/journal-fold-undo.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts index 41cc727a279c..a114117ef071 100644 --- a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts @@ -207,6 +207,17 @@ describe('a failed append leaves the fold equal to a replay of the disk', () => expect(violations).toEqual([]) }) + it("keeps a person's Stop and the queue's Resume as a replay reads them", async () => { + const journal = await open('stop') + await journal.appendItem(codexItem('t-1', 0), assistant('x'), THREAD) + const violations = failEveryAppendOnce(journal, 'stop') + + await journal.appendStopEvent({ reason: 'user-stop', turnId: 't-1' }, CORPUS_FENCE) + await journal.appendQueueResume(CORPUS_FENCE) + await journal.appendStopEvent({ reason: 'user-stop' }, CORPUS_FENCE) + expect(violations).toEqual([]) + }) + it.each(JOURNAL_SESSION_STATE_CASES)( 'with a bookkeeping savepoint that always fails: %s', async (name) => { From df278a75114f4e3ed45da6aae7e8d89202724e21 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:44:33 -0700 Subject: [PATCH 050/107] test(native-chat): an older schema's build reads a version-5 journal read-only; the Stop still pauses after the round trip --- ...-session-stop-event-downgrade.unit.test.ts | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts index 20b64c7f6dbb..b65a7331a1e5 100644 --- a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts +++ b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts @@ -20,9 +20,10 @@ import { importReleaseCheckoutModule, materializeReleaseCheckout } from './relea // make it delete the journal from that row on, so both ride a tombstone it already reads. const BASELINE_REF = 'v1.4.218' const JOURNAL = 'src/main/native-chat/agent-session-journal' -// A main build that shares this one's host database and schema version, so a downgrade to it opens -// the journal writable. No release tag has that database yet; move to the first one that does. -const WRITABLE_BASELINE_REF = '3727100cc9dbcea6201f8a3e506676a3c4b53b18' +// A main build that shares this one's host database at the schema version before 5. Version 5 added +// each chat's stored status, so this build's database opens read-only there. Once a build at +// version 5 exists, pin it here and assert the writable round trip again. +const OLDER_SCHEMA_BASELINE_REF = '3727100cc9dbcea6201f8a3e506676a3c4b53b18' const IDENTITY: AgentSessionJournalIdentity = { sessionId: 'session-downgrade', @@ -173,7 +174,7 @@ function storedRows(directory: string): string[] { } } -test("an older build opens this build's journal writable and appends to it; the pause survives the round trip", async () => { +test("an older schema's build reads this build's journal read-only, writes nothing, and the pause survives the round trip", async () => { const directory = mkdtempSync(join(tmpdir(), 'orca-stop-event-writable-downgrade-')) const journals = createTrackedJournalOpener() const itemIds = (journal: Pick) => @@ -191,7 +192,7 @@ test("an older build opens this build's journal writable and appends to it; the await journals.closeAll() const rowsBefore = storedRows(directory) - const checkout = await materializeReleaseCheckout(WRITABLE_BASELINE_REF) + const checkout = await materializeReleaseCheckout(OLDER_SCHEMA_BASELINE_REF) const support = await importReleaseCheckoutModule( checkout, `${JOURNAL}/journal-host-database-test-support.ts` @@ -199,22 +200,22 @@ test("an older build opens this build's journal writable and appends to it; the const older = releaseExport<() => OlderOpener>(support, 'createTrackedJournalOpener')() try { const downgraded = await older.open({ identity: IDENTITY, stateDirectory: directory }) - expect(downgraded.isReadOnly).toBe(false) + expect(downgraded.isReadOnly).toBe(true) expect(downgraded.cursor()).toEqual(wrote.cursor) expect(itemIds(downgraded)).toEqual(wrote.items) - await downgraded.appendItem(item(2), { kind: 'status', text: 'the older build' }, scope) + await expect( + downgraded.appendItem(item(2), { kind: 'status', text: 'the older build' }, scope) + ).rejects.toThrow() } finally { await older.closeAll() } - const rowsAfter = storedRows(directory) - expect(rowsAfter.slice(0, rowsBefore.length)).toEqual(rowsBefore) - expect(rowsAfter).toHaveLength(rowsBefore.length + 1) + expect(storedRows(directory)).toEqual(rowsBefore) - // Upgraded again: the older build's row folds, and the person's Stop still pauses the queue. + // Upgraded again: the history is as this build left it, and the person's Stop still pauses. const upgraded = await journals.open({ identity: IDENTITY, stateDirectory: directory }) expect(upgraded.isReadOnly).toBe(false) - expect(upgraded.cursor().sequence).toBe(wrote.cursor.sequence + 1) - expect(itemIds(upgraded)).toEqual([...wrote.items, 'codex:thread-1:turn-1:2']) + expect(upgraded.cursor()).toEqual(wrote.cursor) + expect(itemIds(upgraded)).toEqual(wrote.items) expect(upgraded.queuedMessages.pauses('host-a').map((pause) => pause.reason)).toEqual([ 'stopped' ]) From 4c6bf6e41588de9808c60f00a0dfb49c1e763ab5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:05:57 -0700 Subject: [PATCH 051/107] chore: put back main's orcad files the merge's formatter rewrote --- .../ipc/orcad-runtime-maintenance-handlers.ts | 5 +- src/main/ssh/orcad-runtime-decommission.ts | 5 +- src/main/ssh/orcad-runtime-maintenance.ts | 228 +++++++++--------- 3 files changed, 110 insertions(+), 128 deletions(-) diff --git a/src/main/ipc/orcad-runtime-maintenance-handlers.ts b/src/main/ipc/orcad-runtime-maintenance-handlers.ts index 0f16da4de074..1fe7c49cdda8 100644 --- a/src/main/ipc/orcad-runtime-maintenance-handlers.ts +++ b/src/main/ipc/orcad-runtime-maintenance-handlers.ts @@ -23,10 +23,7 @@ export function registerOrcadRuntimeMaintenanceHandlers(options: { }): void { ipcMain.handle( 'runtimeEnvironments:updateOrcad', - async ( - _event, - args: { selector: string; force?: boolean } - ): Promise => { + async (_event, args: { selector: string; force?: boolean }): Promise => { const result = await updateManagedOrcadEnvironment(options.getUserDataPath(), { selector: requiredString(args?.selector, 'Server'), force: args?.force === true diff --git a/src/main/ssh/orcad-runtime-decommission.ts b/src/main/ssh/orcad-runtime-decommission.ts index 4eb9f335fa0a..ffb8d36954cf 100644 --- a/src/main/ssh/orcad-runtime-decommission.ts +++ b/src/main/ssh/orcad-runtime-decommission.ts @@ -7,10 +7,7 @@ import type { OrcadManagedStopResult } from '../../shared/orcad-managed-runtime' import { removeManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' -import type { - KnownRuntimeEnvironment, - OrcadDeploymentLink -} from '../../shared/runtime-environments' +import type { KnownRuntimeEnvironment, OrcadDeploymentLink } from '../../shared/runtime-environments' import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' import { withStaleOrcadActivationRecoveryLock } from './orcad-activation-lock' import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' diff --git a/src/main/ssh/orcad-runtime-maintenance.ts b/src/main/ssh/orcad-runtime-maintenance.ts index 712d0d67ae00..5ed3a8e16f52 100644 --- a/src/main/ssh/orcad-runtime-maintenance.ts +++ b/src/main/ssh/orcad-runtime-maintenance.ts @@ -72,114 +72,106 @@ export function updateManagedOrcadEnvironment( userDataPath: string, args: LifecycleArgs & { force?: boolean } ): Promise { - return withManagedOrcadLifecycle( - userDataPath, - args.selector, - async ({ environment, deployment }) => { - const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) - const census = await collectManagedTerminalCensus( - userDataPath, - environment, - context.activationRecord - ) - const localOrcadDir = await materializeOrcadArtifact(context.serverTarget, { - signal: args.signal - }) - const result = await deployOrcad({ - ...managedOrcadSlot(context, deployment.remotePort, args.signal), - localOrcadDir, - target: context.serverTarget, - census, - force: args.force - }) - if (result.outcome === 'installed-not-activated') { - const deferral = { - outcome: 'deferred' as const, - candidateVersion: result.fullVersion, - code: result.code, - reason: result.reason, - forceable: isForceableOrcadDeferral(result.code) - } - recordManagedOrcadUpdateDeferral(environment.id, deferral) - return deferral - } - clearManagedOrcadUpdateDeferral(environment.id) - const readiness = await probeManagedOrcadReadiness( - context, - localOrcadDir, - result.fullVersion, - args.signal - ) - const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) - return { - outcome: result.outcome === 'already-active' ? 'already-current' : 'updated', - environment: redactRuntimeEnvironment(updated), - activeVersion: result.fullVersion + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const census = await collectManagedTerminalCensus( + userDataPath, + environment, + context.activationRecord + ) + const localOrcadDir = await materializeOrcadArtifact(context.serverTarget, { + signal: args.signal + }) + const result = await deployOrcad({ + ...managedOrcadSlot(context, deployment.remotePort, args.signal), + localOrcadDir, + target: context.serverTarget, + census, + force: args.force + }) + if (result.outcome === 'installed-not-activated') { + const deferral = { + outcome: 'deferred' as const, + candidateVersion: result.fullVersion, + code: result.code, + reason: result.reason, + forceable: isForceableOrcadDeferral(result.code) } + recordManagedOrcadUpdateDeferral(environment.id, deferral) + return deferral } - ) + clearManagedOrcadUpdateDeferral(environment.id) + const readiness = await probeManagedOrcadReadiness( + context, + localOrcadDir, + result.fullVersion, + args.signal + ) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: result.outcome === 'already-active' ? 'already-current' : 'updated', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.fullVersion + } + }) } export function rollbackManagedOrcadEnvironment( userDataPath: string, args: LifecycleArgs ): Promise { - return withManagedOrcadLifecycle( - userDataPath, - args.selector, - async ({ environment, deployment }) => { - const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) - const record = context.activationRecord - const target = record.previous - if (!target) { - return { - outcome: 'refused', - code: 'orcad_rollback_no_target', - reason: 'This server has no previous version to roll back to.' - } - } - const census = await collectManagedTerminalCensus(userDataPath, environment, record) - // Why idle only: this client cannot read the older build's daemon protocol, so it cannot show - // that build would reach terminals that are still running. - if (census.liveSessions !== 0) { - return { - outcome: 'refused', - code: - census.liveSessions === null - ? 'orcad_rollback_census_unavailable' - : 'orcad_rollback_terminals_running', - reason: - census.liveSessions === null - ? 'The server did not answer how many terminals it runs. Retry when it answers.' - : 'Close the terminals running on this server before rolling it back.' - } - } - const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) - const targetDir = managedOrcadInstallDir(context, target) - const targetBuildHash = await readRemoteOrcadBuildHash(slot, targetDir) - const result = await rollbackOrcad({ - ...slot, - record, - census, - targetBuildHash, - targetDaemonProtocol: CURRENT_ORCAD_DAEMON_PROTOCOL - }) - if (result.outcome !== 'rolled-back') { - return result + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const record = context.activationRecord + const target = record.previous + if (!target) { + return { + outcome: 'refused', + code: 'orcad_rollback_no_target', + reason: 'This server has no previous version to roll back to.' } - const readiness = await probeActiveOrcadReadiness( - { ...slot, remoteInstallDir: targetDir }, - { buildHash: targetBuildHash, fullVersion: result.target } - ) - const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + } + const census = await collectManagedTerminalCensus(userDataPath, environment, record) + // Why idle only: this client cannot read the older build's daemon protocol, so it cannot show + // that build would reach terminals that are still running. + if (census.liveSessions !== 0) { return { - outcome: 'rolled-back', - environment: redactRuntimeEnvironment(updated), - activeVersion: result.target, - discarded: result.discarded + outcome: 'refused', + code: + census.liveSessions === null + ? 'orcad_rollback_census_unavailable' + : 'orcad_rollback_terminals_running', + reason: + census.liveSessions === null + ? 'The server did not answer how many terminals it runs. Retry when it answers.' + : 'Close the terminals running on this server before rolling it back.' } } - ) + const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) + const targetDir = managedOrcadInstallDir(context, target) + const targetBuildHash = await readRemoteOrcadBuildHash(slot, targetDir) + const result = await rollbackOrcad({ + ...slot, + record, + census, + targetBuildHash, + targetDaemonProtocol: CURRENT_ORCAD_DAEMON_PROTOCOL + }) + if (result.outcome !== 'rolled-back') { + return result + } + const readiness = await probeActiveOrcadReadiness( + { ...slot, remoteInstallDir: targetDir }, + { buildHash: targetBuildHash, fullVersion: result.target } + ) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: 'rolled-back', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.target, + discarded: result.discarded + } + }) } /** Finishes or undoes an interrupted activation, rollback or decommission on the host. */ @@ -187,29 +179,25 @@ export function recoverManagedOrcadEnvironment( userDataPath: string, args: LifecycleArgs ): Promise { - return withManagedOrcadLifecycle( - userDataPath, - args.selector, - async ({ environment, deployment }) => { - const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) - const result = await recoverInterruptedOrcadActivation( - managedOrcadSlot(context, deployment.remotePort, args.signal) - ) - if (result.outcome !== 'recovered') { - return result - } - const updated = result.readiness - ? refreshPairing(userDataPath, environment, result.readiness, deployment.localPort) - : environment - if (result.activeVersion) { - await ensureOrcadManagedTunnel(userDataPath, environment.id) - } - return { - outcome: 'recovered', - resolution: result.resolution, - activeVersion: result.activeVersion, - environment: redactRuntimeEnvironment(updated) - } + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const result = await recoverInterruptedOrcadActivation( + managedOrcadSlot(context, deployment.remotePort, args.signal) + ) + if (result.outcome !== 'recovered') { + return result } - ) + const updated = result.readiness + ? refreshPairing(userDataPath, environment, result.readiness, deployment.localPort) + : environment + if (result.activeVersion) { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + } + return { + outcome: 'recovered', + resolution: result.resolution, + activeVersion: result.activeVersion, + environment: redactRuntimeEnvironment(updated) + } + }) } From 52c6b86ebd2908570f355b197d32370aeb83dcda Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:22:44 -0700 Subject: [PATCH 052/107] fix(native-chat): a failed chat startup step reaches the diagnostics trace --- src/main/orcad/orcad-entry.ts | 4 +- src/main/orcad/orcad-push-startup.test.ts | 2 +- ...structured-agent-session-create-support.ts | 64 ++++++++----------- ...-agent-session-startup-restoration.test.ts | 23 +++++++ .../structured-agent-session-startup-step.ts | 45 +++++++++++++ ...-process-runtime-launch-activation.test.ts | 5 +- .../startup/main-process-runtime-launch.ts | 6 +- 7 files changed, 104 insertions(+), 45 deletions(-) diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index c2f201566ada..ce2b4b3acd6d 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -312,9 +312,7 @@ async function startOrcadRuntime( // settles crashed ones itself, before any client connects. Chat commands a client sends meanwhile // wait for that settle. runtime.holdStructuredAgentSessionCommandsForStartup() - void runtime.prepareStructuredAgentSessionStartupRestoration().catch((error: unknown) => { - console.warn('[structured-agent-session] startup restoration failed', error) - }) + runtime.prepareStructuredAgentSessionStartupRestorationAfter(Promise.resolve()) const bindHost = resolveOrcadBindHost(options.bind) const rpc = new OrcaRuntimeRpcServer({ diff --git a/src/main/orcad/orcad-push-startup.test.ts b/src/main/orcad/orcad-push-startup.test.ts index e05c2aa6322d..b2d79c461fc4 100644 --- a/src/main/orcad/orcad-push-startup.test.ts +++ b/src/main/orcad/orcad-push-startup.test.ts @@ -87,7 +87,7 @@ vi.mock('../runtime/orca-runtime', () => ({ rehydrateClientHostedBrowserPages() {} async refreshRestoredOrchestrationAuthority() {} async reconcileLegacyWorkerTerminals() {} - async prepareStructuredAgentSessionStartupRestoration() {} + prepareStructuredAgentSessionStartupRestorationAfter() {} holdStructuredAgentSessionCommandsForStartup() {} async stopLegacyWorkerTerminalRecovery() {} setMobilePushRegistrar( diff --git a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts index 0505c1e67f81..7bfccb509390 100644 --- a/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts +++ b/src/main/runtime/orca-runtime-get-structured-agent-session-create-support.ts @@ -10,7 +10,7 @@ import { resolveStructuredAgentSessionAdoptionForCreate } from './structured-agent-session-create-adoption' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' -import { runStructuredAgentSessionStartupStep } from './structured-agent-session-startup-step' +import { runStructuredAgentSessionStartup } from './structured-agent-session-startup-step' import { getLocalProjectWorktreeGitOptions } from '../project-runtime-git-options' import type { AgentSessionAttachParams } from '../native-chat/agent-session-wire/structured-agent-session-attach' import { resolveTuiAgentLaunchEnv } from '../../shared/tui-agent-launch-defaults' @@ -20,12 +20,11 @@ import { } from './structured-agent-account-home' import { resolveStructuredLaunchSeedOptions } from '../../shared/native-chat-session-option-defaults' import { hasPersistedStructuredAgentSessionStore as hasPersistedStructuredAgentSessionStoreOnDisk } from './structured-agent-session-runtime' -import { ensureStructuredAgentSessionHostUnlessRefused } from './structured-agent-session-host-refusal' import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' import { parseWslUncPath } from '../../shared/wsl-paths' import { parseWorkspaceKey } from '../../shared/workspace-scope' import { applyStructuredCodexWorkspaceTrust } from '../agent-workspace-trust-spawn' -import type { StartupStepOutcome } from './structured-agent-session-startup-gate' +import { createStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaRuntimeWithGetWorktreePs { // The history restore a tab restore owes, until a caller that answered with its list starts it. @@ -33,6 +32,7 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR // Listed chats startup could not answer from stored state; null until it has run. protected structuredAgentSessionBackgroundRestoreIds: string[] | null = null protected structuredAgentSessionStartupStepPromise: Promise | null = null + private readonly structuredAgentSessionStartupLogger = createStructuredAgentSessionLogger() async getStructuredAgentSessionCreateSupport( worktreeSelector: string, @@ -295,7 +295,23 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR startStructuredAgentSessionStartupAfter(shellPathReady: Promise): void { void shellPathReady .then(() => this.startStructuredAgentSessionStartup()) - .catch((error: unknown) => console.warn('[structured-agent-session] startup failed', error)) + .catch(this.reportStructuredAgentSessionStartupFailure) + } + + /** `prepare` once `after` resolves, its failure reported rather than thrown: desktop runs it once + * the first window's services are up or timed out, orcad at once. */ + prepareStructuredAgentSessionStartupRestorationAfter(after: Promise): void { + void after + .then(() => this.prepareStructuredAgentSessionStartupRestoration()) + .catch(this.reportStructuredAgentSessionStartupFailure) + } + + /** A failed host build or seed at launch: in the diagnostics trace, not only the console. */ + private reportStructuredAgentSessionStartupFailure = (error: unknown): void => { + this.structuredAgentSessionStartupLogger.warn('the chat startup step failed', { + scope: 'startup-step-failed', + error + }) } /** The host's startup step, once: the host build, then the lease check, seed and settle. */ @@ -309,38 +325,14 @@ export class OrcaRuntimeWithGetStructuredAgentSessionCreateSupport extends OrcaR } protected async startStructuredAgentSessionStartupOnce(): Promise { - const gate = this.structuredAgentSessionStartupGate - gate.stepStarted() - // Chat commands held for startup go ahead however this ends: when the settle does, or now. - let ended: StartupStepOutcome | null = 'no chats on disk' - try { - if (!this.hasPersistedStructuredAgentSessionStore()) { - return - } - ended = 'no host' - // A refused host is no host: startup goes on, and only structured requests are refused. - await ensureStructuredAgentSessionHostUnlessRefused(() => - this.ensureStructuredAgentSessionHost() - ) - const host = getStructuredAgentSessionHost() - if (host) { - this.structuredAgentSessionBackgroundRestoreIds = - await runStructuredAgentSessionStartupStep( - host, - this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null, - (settled) => { - ended = null - gate.openWhen(settled) - } - ) - } - } catch (error) { - ended &&= 'step failed' - throw error - } finally { - if (ended) { - gate.stepEnded(ended) - } + const background = await runStructuredAgentSessionStartup({ + gate: this.structuredAgentSessionStartupGate, + hasChatsOnDisk: () => this.hasPersistedStructuredAgentSessionStore(), + buildHost: () => this.ensureStructuredAgentSessionHost(), + savedSession: () => this.store?.getWorkspaceSession?.(LOCAL_EXECUTION_HOST_ID) ?? null + }) + if (background) { + this.structuredAgentSessionBackgroundRestoreIds = background } } diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index 74618512bdb4..85417c88aca5 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -21,9 +21,11 @@ import { } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-observations' import { OrcaRuntimeService } from './orca-runtime' import { StructuredAgentSessionStartupGate } from './structured-agent-session-startup-gate' +import type { StructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' type PrepareInternals = { structuredAgentSessionStartupGate: StructuredAgentSessionStartupGate + structuredAgentSessionStartupLogger: StructuredAgentSessionLogger store: { getWorkspaceSession: () => unknown } hasPersistedStructuredAgentSessionStore(): boolean refreshMobileSessionPtyRecords(): Promise | null> @@ -126,6 +128,27 @@ it('lets held commands go before the shortest client timeout on a held call (the } }) +it('reports a failed startup step to the diagnostics logger, not only the console', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.spyOn(console, 'info').mockImplementation(() => undefined) + const runtime = restartedRuntime() + const internal = internals(runtime) + const failure = new Error('host build failed') + internal.ensureStructuredAgentSessionHost = async () => { + throw failure + } + const warn = vi.spyOn(internal.structuredAgentSessionStartupLogger, 'warn') + + runtime.startStructuredAgentSessionStartupAfter(Promise.resolve()) + + await vi.waitFor(() => + expect(warn).toHaveBeenCalledWith('the chat startup step failed', { + scope: 'startup-step-failed', + error: failure + }) + ) +}) + it('closes the gate once per launch: a hold after it opened leaves it open', async () => { const gate = new StructuredAgentSessionStartupGate() gate.hold() diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index c00d2c8aef08..f8b95b44f438 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -4,7 +4,13 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' +import { ensureStructuredAgentSessionHostUnlessRefused } from './structured-agent-session-host-refusal' +import type { + StartupStepOutcome, + StructuredAgentSessionStartupGate +} from './structured-agent-session-startup-gate' /** Named from the host so a rename fails to compile. */ export type StructuredAgentSessionStartupHost = Pick< @@ -58,3 +64,42 @@ export function listedStructuredAgentSessionIds( ]) ] } + +/** + * The whole startup step, timed on the gate: the host build, then the step above. Chat commands + * held for startup go ahead however this ends: when the settle does, or as it returns or throws. + * Answers the listed chats the background restore still opens, or null when there is no host. + */ +export async function runStructuredAgentSessionStartup(input: { + gate: StructuredAgentSessionStartupGate + hasChatsOnDisk: () => boolean + buildHost: () => Promise + savedSession: () => WorkspaceSessionState | null +}): Promise { + const { gate } = input + gate.stepStarted() + let ended: StartupStepOutcome | null = 'no chats on disk' + try { + if (!input.hasChatsOnDisk()) { + return null + } + ended = 'no host' + // A refused host is no host: startup goes on, and only structured requests are refused. + await ensureStructuredAgentSessionHostUnlessRefused(input.buildHost) + const host = getStructuredAgentSessionHost() + if (!host) { + return null + } + return await runStructuredAgentSessionStartupStep(host, input.savedSession(), (settled) => { + ended = null + gate.openWhen(settled) + }) + } catch (error) { + ended &&= 'step failed' + throw error + } finally { + if (ended) { + gate.stepEnded(ended) + } + } +} diff --git a/src/main/startup/main-process-runtime-launch-activation.test.ts b/src/main/startup/main-process-runtime-launch-activation.test.ts index 5298ec5feca2..92884fb10136 100644 --- a/src/main/startup/main-process-runtime-launch-activation.test.ts +++ b/src/main/startup/main-process-runtime-launch-activation.test.ts @@ -153,10 +153,11 @@ describe('desktop startup activation', () => { launchHooks.failBeforeWindow = false state.mainWindow = null state.isServeMode = false - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime, holds chat commands and starts the chat startup step before the mocked RPC server takes it. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only null-checks the runtime, holds chat commands and starts the chat startup step and its tab-restore preparation before the mocked RPC server takes it. state.runtime = { holdStructuredAgentSessionCommandsForStartup: () => undefined, - startStructuredAgentSessionStartupAfter: () => undefined + startStructuredAgentSessionStartupAfter: () => undefined, + prepareStructuredAgentSessionStartupRestorationAfter: () => undefined } as unknown as NonNullable // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch only calls whenReady(). state.windowsShellPathHydration = { diff --git a/src/main/startup/main-process-runtime-launch.ts b/src/main/startup/main-process-runtime-launch.ts index 808ca51d9b08..1cf035323ad3 100644 --- a/src/main/startup/main-process-runtime-launch.ts +++ b/src/main/startup/main-process-runtime-launch.ts @@ -345,9 +345,9 @@ export async function initializeMainProcessRuntimeLaunch( } // The tab restore's terminal records refresh lists the daemon's terminals. The first-window // services resolve when the daemon and hook server are up, or at their 12 s timeout regardless. - void state.firstWindowStartupServicesReady - .then(() => runtime.prepareStructuredAgentSessionStartupRestoration()) - .catch((error: unknown) => console.warn('[structured-agent-session] startup failed', error)) + runtime.prepareStructuredAgentSessionStartupRestorationAfter( + state.firstWindowStartupServicesReady + ) app.on('activate', options.handleMacAppActivation) if (serveOptions) { await launchServeMode(runtime, runtimeRpc, serveOptions) From a1e4852db5b0e2a1973a824c27202ccaa81f4b2d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:31:09 -0700 Subject: [PATCH 053/107] perf(native-chat): the status feed reads the newest accepted send from the submissions, not a render of the journal --- .../first-work-branch-rename.test.ts | 8 +-- .../claude-background-wake-retention.test.ts | 4 +- ...ed-agent-session-status-child-work.test.ts | 53 +++++++++++++++++++ ...uctured-agent-session-status-child-work.ts | 14 ++++- ...ructured-agent-session-status-feed.test.ts | 5 +- ...agent-session-status-journal-projection.ts | 14 ++--- ...ructured-agent-session-rpc.test-fixture.ts | 4 +- 7 files changed, 83 insertions(+), 19 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.test.ts diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index 2421ba440816..a043ef1572d6 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -105,9 +105,9 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { // A real journal's sequence only ever advances, so the feed's projection // cache must miss on every publish here: this test is about the rename. let sequence = 0 - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (snapshot, cursor, its projection, activity, read-only). + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (submissions, cursor, its projection, activity, read-only). const journal = { - snapshot: () => ({ items }), + submissions: () => [], lastActivityAt: () => 1, isReadOnly: false, cursor: () => ({ epoch: 1, sequence: (sequence += 1) }), @@ -215,13 +215,13 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { } } ] - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (snapshot, cursor, its projection, activity, read-only). + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (submissions, cursor, its projection, activity, read-only). const journal = { isReadOnly: false, lastActivityAt: () => 1, cursor: () => ({ epoch: 1, sequence: 1 }), statusState: (fence?: number) => projectStructuredAgentSessionStatusState(items, [], fence), - snapshot: () => ({ items }) + submissions: () => [] } as unknown as AgentSessionJournal const location = { executionHostId: 'local' as const, diff --git a/src/main/claude/claude-background-wake-retention.test.ts b/src/main/claude/claude-background-wake-retention.test.ts index b6d4c53daa1b..7667126e75e6 100644 --- a/src/main/claude/claude-background-wake-retention.test.ts +++ b/src/main/claude/claude-background-wake-retention.test.ts @@ -40,7 +40,7 @@ async function wiredSession() { cursor: () => ({ epoch: 'epoch-1', sequence: ++sequence }), isReadOnly: false, lastActivityAt: () => 1, - snapshot, + submissions: () => snapshot().submissions, // The journal's own projection, which the feed shares with the status stored beside it. statusState: (fence?: number) => { const { items, submissions: sent } = snapshot() @@ -53,7 +53,7 @@ async function wiredSession() { [ parent.sessionId, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the feed reads only the cursor, read-only flag, activity clock, snapshot and projection served here. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the feed reads only the cursor, read-only flag, activity clock, submissions and projection served here. journal: journal as unknown as Journal, params: { location: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.test.ts new file mode 100644 index 000000000000..54a8a625c857 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.test.ts @@ -0,0 +1,53 @@ +// The user's newest accepted send, read from the journal's submissions without rendering it. + +import { describe, expect, it } from 'vitest' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { newestAcceptedSendKey } from './structured-agent-session-status-child-work' + +function submission( + index: number, + submittedAt: number, + dispatchState: AgentJournalSubmission['dispatchState'] +): AgentJournalSubmission { + return { + clientMessageId: `send-${index}`, + fence: 1, + payloadFingerprint: `fingerprint-${index}`, + dispatchState, + providerItemId: null, + reason: null, + submittedAt, + resolvedAt: null + } +} + +/** The key as read from the journal's render: its submissions by `submittedAt` (a stable sort), + * then the last one accepted. */ +function renderedKey(submissions: readonly AgentJournalSubmission[]): string { + const rendered = [...submissions].sort((a, b) => a.submittedAt - b.submittedAt) + return `epoch:${rendered.findLast((entry) => entry.dispatchState === 'accepted')?.clientMessageId ?? ''}` +} + +describe('newestAcceptedSendKey', () => { + it('reads from the submissions in fold order what it reads from the rendered list', () => { + let seed = 7 + const next = (bound: number) => { + seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648 + return seed % bound + } + const states: AgentJournalSubmission['dispatchState'][] = ['accepted', 'pending', 'rejected'] + for (let round = 0; round < 300; round += 1) { + // Few distinct clocks, so ties are common; fold order is insertion order, not clock order. + const submissions = Array.from({ length: next(8) }, (_, index) => + submission(index, next(4), states[next(states.length)] ?? 'accepted') + ) + expect(newestAcceptedSendKey('epoch', submissions)).toBe(renderedKey(submissions)) + } + }) + + it('names the later of two accepted sends with the same clock, as the render orders them', () => { + const submissions = [submission(0, 5, 'accepted'), submission(1, 5, 'accepted')] + + expect(newestAcceptedSendKey('epoch', submissions)).toBe('epoch:send-1') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts index 10a447292627..327b57195836 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-child-work.ts @@ -58,8 +58,18 @@ export function structuredStatusChildrenEqual( * never one; a rewind replaces the epoch. */ export function newestAcceptedSendKey( epoch: string, - submissions: readonly AgentJournalSubmission[] + submissions: Iterable ): string { - const accepted = submissions.findLast((submission) => submission.dispatchState === 'accepted') + // Newest by `submittedAt`, a tie to the later one: what the last accepted send of the journal's + // render (sorted by `submittedAt`, stable) is, without rendering the journal. + let accepted: AgentJournalSubmission | undefined + for (const submission of submissions) { + if ( + submission.dispatchState === 'accepted' && + (!accepted || submission.submittedAt >= accepted.submittedAt) + ) { + accepted = submission + } + } return `${epoch}:${accepted?.clientMessageId ?? ''}` } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index 42f103dcc9cc..c4b2dbd4c1bf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -659,14 +659,15 @@ describe('StructuredAgentSessionStatusFeed', () => { expect(feed.statusState(SESSION)).toBe(projected) } expect(events).toHaveLength(101) - expect(snapshot).toHaveBeenCalledTimes(1) + // The row never renders the journal: the projection and the accepted send come without it. + expect(snapshot).not.toHaveBeenCalled() expect(events.at(-1)).toMatchObject({ type: 'status', session: { status: 'working', backgroundTasks: [{ state: 'working' }] } }) await journal.appendTombstone(TURN_IDENTITY, { fence: 1 }) feed.publish(SESSION) - expect(snapshot).toHaveBeenCalledTimes(2) + expect(snapshot).not.toHaveBeenCalled() expect(feed.statusState(SESSION)).not.toBe(projected) expect(events.at(-1)).toMatchObject({ type: 'status', session: { status: 'idle' } }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index 128aff708433..ffed51c368d2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -46,18 +46,18 @@ export class StructuredAgentSessionJournalProjections { ) { // A journalled submission bumps `lastSequence`, so the send-time working // signal reaches the cache; the lease fence does not, hence the extra key. - const snapshot = readOnly ? null : journal.snapshot() projection = { ...cursor, readOnly, fence, // The journal's own projection, shared with the status it stores beside each write. - state: snapshot - ? journal.statusState(fence) - : projectStructuredAgentSessionStatusState([], [], fence), - acceptedSendKey: snapshot - ? newestAcceptedSendKey(cursor.epoch, snapshot.submissions ?? []) - : null + state: readOnly + ? projectStructuredAgentSessionStatusState([], [], fence) + : journal.statusState(fence), + // From the submissions alone: rendering the whole journal for one key costs every commit. + acceptedSendKey: readOnly + ? null + : newestAcceptedSendKey(cursor.epoch, journal.submissions()) } this.byJournal.set(journal, projection) } diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index 46cb1e8c4d22..fead570d1ee0 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -102,12 +102,12 @@ function statusFeed(): StructuredAgentSessionStatusFeed { [ STATUS_SESSION, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (snapshot, cursor, its projection, activity, read-only). + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a stand-in journal offering only what the status feed reads (submissions, cursor, its projection, activity, read-only). journal: { isReadOnly: false, cursor: () => ({ epoch: 'epoch-status', sequence: 2 }), lastActivityAt: () => 2, - snapshot: () => ({ items: STATUS_ITEMS }), + submissions: () => [], statusState: (fence?: number) => projectStructuredAgentSessionStatusState(STATUS_ITEMS, [], fence) } as unknown as AgentSessionJournal, From 913fc6f8a088d024ea186fc406b8ebd51e0e1cfa Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:36:26 -0700 Subject: [PATCH 054/107] test(native-chat): a turn a person's Stop found settles as theirs, from death evidence observed after the Stop --- .../journal-session-state-test-corpus.ts | 20 +++++++++- ...ured-agent-session-open-settlement.test.ts | 39 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts index a81a68c09c46..e822877dd5c2 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts @@ -203,6 +203,16 @@ export const JOURNAL_SESSION_STATE_CORPUS = { await settledTurn(journal, 'turn-1') await send(journal, 'send-recovered', 'lost in a crash') await journal.markPendingSubmissionsUnknown(CORPUS_FENCE) + }, + // A person's Stop found the turn running, then Orca died before the provider ended it. + "running turn a person's Stop found": async (journal: AgentSessionJournal) => { + await item(journal, codexItem('turn-2', 0), { + kind: 'turn', + turnId: 'turn-2', + state: 'running', + startedAt: 10 + }) + await journal.appendStopEvent({ reason: 'user-stop', turnId: 'turn-2' }, CORPUS_FENCE) } } satisfies Record Promise> @@ -229,7 +239,8 @@ export const CORPUS_UNSETTLED: Record = { 'refused send': false, 'running turn on a legacy status row': true, 'running work settled by a batch': false, - 'unknown send recovered': false + 'unknown send recovered': false, + "running turn a person's Stop found": true } export const JOURNAL_SESSION_STATE_CASES = Object.keys(JOURNAL_SESSION_STATE_CORPUS).filter( @@ -252,5 +263,12 @@ export const CORPUS_DEATH_EVIDENCE: Record { ).toBe(true) }) + it("ends a turn a person's Stop found as theirs, with no row saying the provider stopped", async () => { + const providerStoppedRows = (plan: ReturnType) => + (plan.goneGeneration?.mutations ?? []).filter( + (mutation) => + mutation.kind === 'item' && + mutation.identity.provider === 'orca' && + mutation.identity.clientMessageId.includes(':death-') + ) + const settle = async (evidenceName: string) => { + const sessionId = `chat-${(chats += 1)}` + const journal = await open(sessionId) + await JOURNAL_SESSION_STATE_CORPUS["running turn a person's Stop found"](journal) + const deathEvidence = CORPUS_DEATH_EVIDENCE[evidenceName] ?? null + const record: OpenSettlementRecordFacts = { sessionId, fence: CORPUS_FENCE, deathEvidence } + const plan = planOpenSettlement(journal, record, { settlesRosters: true }) + await appendOpenSettlement(journal, plan, CORPUS_FENCE, (error) => { + throw error + }) + return { sessionId, journal, plan } + } + + // The process was found dead after the Stop: the Stop decides how the turn ends. + const after = await settle('names the writer, after a Stop') + expect(providerStoppedRows(after.plan)).toEqual([]) + expect(after.plan.goneGeneration?.mutations).toContainEqual( + expect.objectContaining({ + body: expect.objectContaining({ kind: 'turn', state: 'interrupted' }) + }) + ) + expect(selected(after.sessionId)).toBe(false) + expect(storedStatus(after.sessionId).summary).toEqual( + after.journal.statusState(undefined).summary + ) + + // Found dead before the Stop event's time: the death explains the end, and says so. + const before = await settle('names the writer') + expect(providerStoppedRows(before.plan)).toHaveLength(1) + }) + it('agrees on a chat that opened corrupt, and again once it writes past the repair', async () => { const first = await open('corrupt') await JOURNAL_SESSION_STATE_CORPUS['working subagent roster'](first) From 4d25c777973ed946a0c9bd725845684e76050d5c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:13:32 -0700 Subject: [PATCH 055/107] perf(native-chat): rowless listed chats get their status from their rows, with no conversation opened --- .../journal-row-table.ts | 7 + .../journal-session-status-backfill.test.ts | 172 ++++++++++++++++++ .../journal-session-status-backfill.ts | 127 +++++++++++++ .../structured-agent-session-reveal.ts | 31 ++-- ...ctured-agent-session-startup-state.test.ts | 10 +- .../structured-agent-session-startup-state.ts | 76 +++++++- ...ctured-agent-session-status-derive.test.ts | 153 ++++++++++++++++ 7 files changed, 560 insertions(+), 16 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-row-table.ts b/src/main/native-chat/agent-session-journal/journal-row-table.ts index 4665a0571150..266e372723a9 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-table.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-table.ts @@ -10,6 +10,7 @@ import { serializeJournalRow, type JournalRow } from './journal-row-schema' export type JournalStoredRow = { epoch: string; seq: number; ts: number; rowJson: string } const SELECT_EPOCH = 'SELECT epoch FROM journal_sessions WHERE session_id = ?' +const SELECT_TIP = 'SELECT max(seq) AS tip FROM journal_rows WHERE session_id = ? AND epoch = ?' const PUBLISH_SESSION_EPOCH = `INSERT INTO journal_sessions (session_id, workspace_id, epoch) VALUES (?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET @@ -29,6 +30,12 @@ export function readJournalSessionEpoch(db: Database.Database, sessionId: string return typeof epoch === 'string' ? epoch : null } +/** The highest sequence stored under `epoch`, or 0 with none. */ +export function readJournalTip(db: Database.Database, sessionId: string, epoch: string): number { + const tip = db.prepare(SELECT_TIP).get(sessionId, epoch)?.tip + return typeof tip === 'number' ? tip : 0 +} + /** Points the chat at `epoch`. Only an epoch change writes this row. */ export function publishJournalSessionEpoch( db: Database.Database, diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts new file mode 100644 index 000000000000..48988748c6dd --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -0,0 +1,172 @@ +// A missing status row written from the chat's rows alone, with no open: the row an open would +// write, folded a bounded part per task, and nothing written when the chat moved or quit stops it. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { + createTrackedJournalOpener, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import * as JournalOpen from './journal-open' +import { backfillJournalSessionStatus } from './journal-session-status-backfill' +import { + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS +} from './journal-session-state-test-corpus' +import type { AgentSessionJournal } from './journal-store' + +vi.mock('./journal-open', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, startJournalRowFold: vi.fn(actual.startJournalRowFold) } +}) + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +function identity(sessionId: string): AgentSessionJournalIdentity { + return { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + } +} + +function open(sessionId: string): Promise { + return journals.open({ + identity: identity(sessionId), + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}-${clock}`, + currentFence: () => CORPUS_FENCE + }) +} + +const database = () => openTestJournalHostDatabase(root) +const dropRow = (sessionId: string) => + database().db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-status-backfill-')) + clock = 1_000 +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe('a row from the rows alone', () => { + it.each(JOURNAL_SESSION_STATE_CASES)('equals the row an open writes: %s', async (name) => { + const journal = await open(name) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + await journals.closeAll() + dropRow(name) + // What an open of the chat writes for a missing row. + const reopened = await open(name) + reopened.backfillSessionStatus() + const byOpen = readTestJournalSessionStatus(root, name) + await journals.closeAll() + dropRow(name) + + const derived = await backfillJournalSessionStatus(database(), name) + + expect(derived?.status ?? null).toEqual(byOpen) + expect(readTestJournalSessionStatus(root, name)).toEqual(byOpen) + }) + + it('folds a large chat a bounded part per task', async () => { + const journal = await open('large') + const text = 'x'.repeat(2_000) + for (let index = 0; index < 200; index += 1) { + await journal.appendItem( + { provider: 'orca', clientMessageId: `note-${index}` }, + { kind: 'status', text: `${index} ${text}` }, + { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } + ) + } + await journals.closeAll() + dropRow('large') + // Characters of row JSON folded since the last yield: what one task does. + let sinceYield = 0 + let largest = 0 + const actual = await vi.importActual('./journal-open') + vi.mocked(JournalOpen.startJournalRowFold).mockImplementationOnce((input) => { + const fold = actual.startJournalRowFold(input) + return { + add: (entry) => { + sinceYield += entry.rowJson.length + return fold.add(entry) + }, + finish: fold.finish + } + }) + const budget = 20_000 + + const derived = await backfillJournalSessionStatus(database(), 'large', { + batchRows: 64, + batchChars: budget, + yieldTask: async () => { + largest = Math.max(largest, sinceYield) + sinceYield = 0 + } + }) + + largest = Math.max(largest, sinceYield) + expect(derived?.status.lifecycle).toBe('idle') + // 200 rows of about 2 KB: many parts, none over the budget. + expect(largest).toBeGreaterThan(budget / 2) + expect(largest).toBeLessThanOrEqual(budget) + }) + + it('writes nothing when the chat moved during the fold, quit stopped it, or it has a row', async () => { + const journal = await open('moving') + await JOURNAL_SESSION_STATE_CORPUS.settled(journal) + dropRow('moving') + // An append lands between parts: the fold read a tip the chat has left. + let appended = false + const moved = await backfillJournalSessionStatus(database(), 'moving', { + batchRows: 1, + yieldTask: async () => { + if (!appended) { + appended = true + dropRow('moving') + await journal.appendItem( + { provider: 'orca', clientMessageId: 'late' }, + { kind: 'status', text: 'late' }, + { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } + ) + dropRow('moving') + } + } + }) + expect(moved).toBeNull() + expect(readTestJournalSessionStatus(root, 'moving')).toBeNull() + + const quit = new AbortController() + const stopped = await backfillJournalSessionStatus(database(), 'moving', { + batchRows: 1, + signal: quit.signal, + yieldTask: async () => quit.abort() + }) + expect(stopped).toBeNull() + expect(readTestJournalSessionStatus(root, 'moving')).toBeNull() + + // Still in a per-chat file, or never written: no epoch here. + expect(await backfillJournalSessionStatus(database(), 'never-written')).toBeNull() + + await backfillJournalSessionStatus(database(), 'moving') + const row = readTestJournalSessionStatus(root, 'moving') + expect(row).not.toBeNull() + // A row already there is never rewritten. + expect(await backfillJournalSessionStatus(database(), 'moving')).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts new file mode 100644 index 000000000000..05a0f9b6bba4 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -0,0 +1,127 @@ +// Writing the status row a chat already in the host's database is missing, without opening it. +// +// Version 5 creates the status table empty, so after an upgrade every chat has no row until +// something writes it. Rather than open each chat (its lease, settle and conversation), its rows are +// folded a bounded part per task, as a replay folds them, then the row is derived and written in one +// short transaction, only while the chat is still where the fold read it. The startup pass and the +// background copy both write missing rows through this one function. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import type { JournalHostDatabase } from './journal-host-database' +import { startJournalRowFold, type JournalLoad } from './journal-open' +import { IMPORT_BATCH_ROWS } from './journal-per-session-source' +import { pendingJournalRepairSequence } from './journal-repair-marker' +import { readJournalRowsAfter, readJournalSessionEpoch, readJournalTip } from './journal-row-table' +import { + deriveJournalSessionStatus, + hasJournalSessionStatus, + writeJournalSessionStatus, + type JournalSessionStatus +} from './journal-session-state' + +/** Row JSON per part, in UTF-16 units: a page of large rows is split, so each main-thread task + * handles about the same bytes (a quarter of a full page of the seed's 2.3 KB rows). */ +export const IMPORT_BATCH_CHARS = 256 * 1024 + +/** Each batch split so no part holds more than `maxChars` of row JSON; a larger row is a part alone. + * Only a batch's last part keeps its `last`. */ +export function* charBoundedBatches( + batches: Iterable<{ rows: Row[]; last: boolean }>, + maxChars = IMPORT_BATCH_CHARS +): Generator<{ rows: Row[]; last: boolean }> { + for (const batch of batches) { + let part: Row[] = [] + let chars = 0 + for (const row of batch.rows) { + if (part.length > 0 && chars + row.rowJson.length > maxChars) { + yield { rows: part, last: false } + part = [] + chars = 0 + } + part.push(row) + chars += row.rowJson.length + } + yield { rows: part, last: batch.last } + } +} + +/** The chat's row: its rows folded a part per task, as a replay folds them, then the row derived + * and written in one short transaction, only while the chat is still where the fold read it. Null + * when something wrote the row or the chat first, the chat has no epoch in this database (it is + * still in a per-chat file), the database or the chat's rows are a newer build's, or `signal` + * aborted, which is checked before each part. */ +export async function backfillJournalSessionStatus( + database: JournalHostDatabase, + sessionId: string, + { + batchRows = IMPORT_BATCH_ROWS, + batchChars = IMPORT_BATCH_CHARS, + yieldTask = () => yieldToEventLoop(), + signal + }: { + batchRows?: number + batchChars?: number + /** Ends each part's task: the next macrotask by default; a background job paces here. */ + yieldTask?: () => Promise + /** Quit: the fold stops within one part and nothing is written. */ + signal?: AbortSignal + } = {} +): Promise<{ load: JournalLoad; status: JournalSessionStatus } | null> { + if (database.readOnly) { + return null + } + const epoch = readJournalSessionEpoch(database.db, sessionId) + if (epoch === null || hasJournalSessionStatus(database.db, sessionId)) { + return null + } + const tip = readJournalTip(database.db, sessionId, epoch) + const fold = startJournalRowFold({ + sessionId, + epoch, + repairedFrom: pendingJournalRepairSequence(database.db, sessionId, epoch) + }) + for (let afterSeq = Number.MIN_SAFE_INTEGER; ;) { + if (signal?.aborted) { + return null + } + const rows = readJournalRowsAfter(database.db, sessionId, epoch, afterSeq, batchRows) + const last = rows.at(-1) + let folding = true + for (const part of charBoundedBatches([{ rows, last: true }], batchChars)) { + folding = part.rows.every(fold.add) + // A part per task: a long chat's whole replay in one task holds up every other chat. + if (!folding || part.last) { + break + } + await yieldTask() + if (signal?.aborted) { + return null + } + } + if (!folding || rows.length < batchRows || !last) { + break + } + afterSeq = last.seq + await yieldTask() + } + if (signal?.aborted) { + return null + } + const load = fold.finish() + // A newer build's rows: as an open of it does, this build writes nothing for the chat. + if (load.readOnly) { + return null + } + return database.transaction((db) => { + if ( + hasJournalSessionStatus(db, sessionId) || + readJournalSessionEpoch(db, sessionId) !== epoch || + readJournalTip(db, sessionId, epoch) !== tip + ) { + return null + } + const status = deriveJournalSessionStatus(load.state, { settlesRosters: !load.corrupt }) + writeJournalSessionStatus(db, sessionId, status) + return { load, status } + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index bcb674ec8069..74cbb175d940 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -110,21 +110,28 @@ export function createStructuredAgentSessionHostRestore( } const restorer = new StructuredAgentSessionReadableRestorer({ ...readRestore, supportsRecord }) const gate = new StructuredAgentSessionRestartRestoreGate() + const startup = createStructuredAgentSessionStartupState({ + openDeps: deps, + canSettle, + seedStatus, + resolveRecovery: readRestore.resolveRecovery, + restoreListed: (records) => + restoreStructuredAgentSessionsOnRestart({ ...readRestore, records }), + serialize: rest.serialize, + hasSession: rest.hasSession, + isDisposed: rest.isDisposed + }) return { reconcileRestartLeases: async () => { await reconcile('startup') }, - restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), - ...createStructuredAgentSessionStartupState({ - openDeps: deps, - canSettle, - seedStatus, - resolveRecovery: readRestore.resolveRecovery, - restoreListed: (records) => - restoreStructuredAgentSessionsOnRestart({ ...readRestore, records }), - serialize: rest.serialize, - hasSession: rest.hasSession, - isDisposed: rest.isDisposed - }) + // The listed chats the tab list left to it: rows derived without an open first, then the rest. + restoreReadableSessions: (sessionIds) => + gate.run(async () => + restorer.restore( + sessionIds === undefined ? undefined : await startup.deriveMissingStatuses(sessionIds) + ) + ), + ...startup } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 0fd5d8bc7991..d0d1e5f7eeac 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -315,8 +315,9 @@ describe('startup opens only what it must (T6, T7, T14)', () => { const background = await startup(rig, listed) const unlisted = ['session-crashed-closed'] + // The rowless chat's row comes from its rows alone: it is never opened. expect(opened(rig, [...listed, ...unlisted]).toSorted()).toEqual( - ['session-crashed', ...unlisted, 'session-rowless', 'session-uncopied'].toSorted() + ['session-crashed', ...unlisted, 'session-uncopied'].toSorted() ) expect(background.toSorted()).toEqual(['session-rowless', 'session-uncopied'].toSorted()) expect(latestRestTestStatus(rig, 'session-draft')).toMatchObject({ status: 'idle' }) @@ -341,8 +342,11 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) } - // Its open wrote the missing status back. - expect(readTestJournalSessionStatus(rig.root, 'session-rowless')).not.toBeNull() + // The missing status was written back, and its row published, with no open. + expect(readTestJournalSessionStatus(rig.root, 'session-rowless')).toMatchObject({ + lifecycle: 'idle' + }) + expect(latestRestTestStatus(rig, 'session-rowless')).toMatchObject({ status: 'idle' }) // The next boot finds nothing to settle and nothing without a status (T4b). await rig.crash() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 7188363cc186..ac5f3a25015c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -23,6 +23,7 @@ import { readJournalSessionStatuses, readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' +import { backfillJournalSessionStatus } from '../agent-session-journal/journal-session-status-backfill' import { openStructuredAgentSessionConversationJournal, type StructuredAgentSessionConversationOpenDeps @@ -71,6 +72,13 @@ export type StructuredAgentSessionStartupState = { * so callers must catch. */ settleClosedChat: (record: AgentSessionRecord) => Promise + /** + * The background pass's first step: each listed chat with history here and no status row gets + * its row from its rows alone, opening nothing, one chat at a time in the order given. Answers + * the chats still to open: the rest, and any whose rows show work a gone process left or owe a + * rebuild, which their open settles or rebuilds. + */ + deriveMissingStatuses: (sessionIds: readonly string[]) => Promise } export function createStructuredAgentSessionStartupState( @@ -83,8 +91,74 @@ export function createStructuredAgentSessionStartupState( settling ??= settleOwedSessions(deps, listedIds) return settling }, - settleClosedChat: (record) => settleClosed(deps, record) + settleClosedChat: (record) => settleClosed(deps, record), + deriveMissingStatuses: (sessionIds) => deriveMissingStatuses(deps, sessionIds) + } +} + +async function deriveMissingStatuses( + deps: StructuredAgentSessionStartupStateDeps, + sessionIds: readonly string[] +): Promise { + if (deps.openDeps.journalDatabase.readOnly) { + return [...sessionIds] + } + // Quit stops a fold within one part, and nothing more is written. + const quit = new AbortController() + const yieldTask = async () => { + await yieldToEventLoop() + if (deps.isDisposed()) { + quit.abort() + } } + const toOpen: string[] = [] + for (const sessionId of sessionIds) { + if (deps.isDisposed()) { + return [] + } + const record = deps.openDeps.store.getRecord(sessionId) + const derived = + record && + deps.canSettle(record) && + !deps.hasSession(sessionId) && + (await deriveFromRows(deps, record, { yieldTask, signal: quit.signal })) + if (!derived) { + toOpen.push(sessionId) + } + } + return toOpen +} + +/** Seeds the chat from a row derived from its rows; false leaves it to its open. */ +async function deriveFromRows( + deps: StructuredAgentSessionStartupStateDeps, + record: AgentSessionRecord, + options: { yieldTask: () => Promise; signal: AbortSignal } +): Promise { + let derived: Awaited> + try { + derived = await backfillJournalSessionStatus( + deps.openDeps.journalDatabase, + record.sessionId, + options + ) + } catch (error) { + deps.openDeps.logger.warn('deriving a chat status from its rows failed', { + scope: 'startup-status-derive', + sessionId: record.sessionId, + error + }) + return false + } + // Work a gone process left is settled, and a corrupt history rebuilt, by the chat's open. + if (!derived || derived.load.corrupt || isUnsettledJournalSessionStatus(derived.status)) { + return false + } + deps.seedStatus(record, { + projected: derived.status.summary, + lastActivityAt: derived.status.lastActivityAt + }) + return true } function seedStoredStatuses( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts new file mode 100644 index 000000000000..e385d48596c8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts @@ -0,0 +1,153 @@ +// The first boot after upgrading: listed chats with history and no status row get their rows from +// their rows alone, one at a time in tab order, with no conversation opened; a chat a crash left +// with work is still opened, so it is settled; and a send during the pass is not queued behind it. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from '../agent-session-journal/journal-host-database-test-support' +import { + createRestTestRig, + restTestChat, + sendRestTestMessage, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { + latestRestTestStatus, + restTestOpens +} from './structured-agent-session-rest-test-observations' + +const rigs: RestTestRig[] = [] + +afterEach(async () => { + for (const rig of rigs.splice(0)) { + await rig.dispose() + } + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +async function newRig(): Promise { + const rig = await createRestTestRig() + rigs.push(rig) + return rig +} + +/** Every chat's row dropped, as version 5 finds a database an older build wrote. */ +function upgradeToEmptyStatusTable(rig: RestTestRig): void { + openTestJournalHostDatabase(rig.root).db.prepare('DELETE FROM journal_session_state').run() +} + +const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds + +/** Startup up to the listing, as the host runs it; answers what the background pass is owed. */ +async function startupThroughListing(rig: RestTestRig): Promise { + const listed = listedIds(rig) + await rig.host.reconcileRestartLeases() + const background = rig.host.seedStoredStatuses(listed) + await rig.host.settleOwedSessions(listed) + return background +} + +const seeded = (rig: RestTestRig, sessionId: string) => + rig.statusEvents.findIndex( + (event) => event.type === 'status' && event.session.sessionId === sessionId + ) + +describe('rowless listed chats after an upgrade', () => { + it('get their rows from their rows alone, in tab order, with no chat opened', async () => { + const rig = await newRig() + const ids = Array.from({ length: 12 }, (_, index) => `session-${index}`) + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + const background = await startupThroughListing(rig) + expect(background).toEqual(listedIds(rig)) + + await rig.host.restoreReadableSessions(background) + + for (const sessionId of ids) { + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) + expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) + expect(restTestOpens(rig, sessionId)).toBe(0) + } + const order = ids.map((sessionId) => seeded(rig, sessionId)) + expect(order).toEqual(order.toSorted((a, b) => a - b)) + }) + + it('still opens, and so settles, a rowless chat a crash left mid-turn', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-fine', { message: 'done' }) + await restTestChat(rig, 'session-crashed', { message: 'asked' }) + const [{ providerIdentity }] = await Promise.all( + rig.adapter.dispatch.mock.results.slice(-1).map((result) => result.value) + ) + await rig.host + .collaboratorsForTests() + .sessions.get('session-crashed')! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { + fence: rig.store.getRecord('session-crashed')!.lease.runtimeFence, + turnScope: { kind: 'thread' } + } + ) + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + + await rig.host.restoreReadableSessions(await startupThroughListing(rig)) + + expect(restTestOpens(rig, 'session-fine')).toBe(0) + expect(restTestOpens(rig, 'session-crashed')).toBeGreaterThan(0) + expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ + lifecycle: 'idle' + }) + }) + + it('accepts a send during the pass with at most one chat derived ahead of it', async () => { + const rig = await newRig() + const ids = Array.from({ length: 6 }, (_, index) => `session-long-${index}`) + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + const journal = rig.host.collaboratorsForTests().sessions.get(sessionId)!.journal + const fence = rig.store.getRecord(sessionId)!.lease.runtimeFence + // Long enough that its fold takes several parts, as a long chat's does. + for (let index = 0; index < 1_200; index += 1) { + await journal.appendItem( + { provider: 'orca', clientMessageId: `${sessionId}-note-${index}` }, + { kind: 'status', text: `${index} ${'x'.repeat(400)}` }, + { fence, turnScope: { kind: 'thread' } } + ) + } + } + await restTestChat(rig, 'session-send', { message: 'first' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + // The chat the send goes to keeps its row; the long ones are the pass's. + await rig.boot() + const background = (await startupThroughListing(rig)).filter((id) => id !== 'session-send') + const derivedCount = () => + ids.filter((sessionId) => readTestJournalSessionStatus(rig.root, sessionId) !== null).length + + const pass = rig.host.restoreReadableSessions(background) + await new Promise((resolve) => setImmediate(resolve)) + const before = derivedCount() + const sent = await sendRestTestMessage(rig, 'session-send', 'during the pass') + const after = derivedCount() + await pass + + expect(sent).toMatchObject({ ok: true }) + expect(after - before).toBeLessThanOrEqual(1) + expect(after).toBeLessThan(ids.length) + expect(derivedCount()).toBe(ids.length) + }, 120_000) +}) From 705624ee5f742be6aadfece1b265323c8e7dcc46 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:14:41 -0700 Subject: [PATCH 056/107] test(native-chat): a status row from a corrupt history's rows equals an open's --- .../journal-session-status-backfill.test.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index 48988748c6dd..9b6da6149cc8 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -8,6 +8,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { createTrackedJournalOpener, + insertTestJournalRowJson, openTestJournalHostDatabase, readTestJournalSessionStatus } from './journal-host-database-test-support' @@ -83,6 +84,27 @@ describe('a row from the rows alone', () => { expect(readTestJournalSessionStatus(root, name)).toEqual(byOpen) }) + it('equals the row an open writes for a chat whose history is corrupt', async () => { + const journal = await open('corrupt') + await JOURNAL_SESSION_STATE_CORPUS['working subagent roster'](journal) + const tip = journal.cursor() + await journals.closeAll() + insertTestJournalRowJson(database().db, 'corrupt', tip.sequence + 1, '{"not a row"') + dropRow('corrupt') + const reopened = await open('corrupt') + reopened.backfillSessionStatus() + const byOpen = readTestJournalSessionStatus(root, 'corrupt') + await journals.closeAll() + dropRow('corrupt') + + const derived = await backfillJournalSessionStatus(database(), 'corrupt') + + // The rebuild the corruption owes decides the roster, so neither row counts it as live work. + expect(derived?.load.corrupt).toBe(true) + expect(byOpen?.liveChildWork).toBe(false) + expect(derived?.status ?? null).toEqual(byOpen) + }) + it('folds a large chat a bounded part per task', async () => { const journal = await open('large') const text = 'x'.repeat(2_000) From 2447f1004339033f48e1f3100b7a3af95fc256c1 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:18:14 -0700 Subject: [PATCH 057/107] perf(native-chat): a status snapshot's rows reach the renderer's store as one publication --- .../activity-answered-ask-timeline.test.tsx | 28 +-- ...tructuredAgentSessionStatusBridge.test.tsx | 124 +++++++------ .../StructuredAgentSessionStatusBridge.tsx | 71 ++++++- ...agent-session-status-bridge-burst.test.tsx | 173 ++++++++++++++++++ ...agent-session-status-bridge-clock.test.tsx | 31 ++-- .../structured-child-records-switch.test.tsx | 14 +- 6 files changed, 345 insertions(+), 96 deletions(-) create mode 100644 src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx diff --git a/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx b/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx index e7ac1d3b7d3e..1ad6587e5eb5 100644 --- a/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx +++ b/src/renderer/src/components/activity/activity-answered-ask-timeline.test.tsx @@ -127,11 +127,11 @@ function thread(activity: ReturnType) { return only } -async function connect(): Promise<(event: AgentSessionStatusEvent) => void> { +async function connect(): Promise<(event: AgentSessionStatusEvent) => Promise> { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const emit: (event: AgentSessionStatusEvent) => void = mocks.subscribeStatus.mock.calls[0]?.[1] - return (event) => act(() => emit(event)) + return (event) => act(async () => emit(event)) } describe("an answered subagent ask on a settled parent's Activity row", () => { @@ -161,15 +161,15 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { it('reads done, lists Blocked then done, and leaves the answer read', async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) acknowledge(SETTLED + 1_000) - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: ASKED, updatedAt: ASKED }) }) // The user reads the ask and answers it; the parent returns to its own turn's end. acknowledge(ASKED + 500) - emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) + await emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) const row = thread(renderActivity()) expect(activityThreadStatusId(row)).toBe('done') @@ -188,7 +188,7 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { expect(countActivityUnread(store().getState())).toBe(0) // Clearing the answered row must also pass the ask, which is dated after the done. - act(() => { + await act(async () => { expect(clearActivityThread(row)).toBe(true) }) expect(renderActivity().result.current.allThreads).toHaveLength(0) @@ -197,13 +197,13 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { it('keeps each answered done between the asks around it', async () => { const asks = [ASKED, ASKED + 400, ASKED + 800] const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) for (const askedAt of asks) { - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: askedAt, updatedAt: askedAt }) }) - emit({ type: 'status', session: summary({ updatedAt: askedAt + 200 }) }) + await emit({ type: 'status', session: summary({ updatedAt: askedAt + 200 }) }) } // Every done repeats the turn's end, so only when each was seen keeps them apart and in order; @@ -221,23 +221,23 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { it('reads done after a clear hid that done, while the later ask stays listed', async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) acknowledge(SETTLED + 1_000) const beforeAsk = renderActivity() let cleared = false - act(() => { + await act(async () => { cleared = clearActivityThread(thread(beforeAsk)) }) expect(cleared).toBe(true) beforeAsk.unmount() expect(store().getState().activityClearedAtByPaneKey).toEqual({ [paneKey()]: SETTLED }) - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: ASKED, updatedAt: ASKED }) }) acknowledge(ASKED + 500) - emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) + await emit({ type: 'status', session: summary({ updatedAt: ANSWERED }) }) const row = thread(renderActivity()) // The cleared done stays cleared; only the ask, seen after the clear, is listed. @@ -248,7 +248,7 @@ describe("an answered subagent ask on a settled parent's Activity row", () => { expect(countActivityUnread(store().getState())).toBe(0) // Once a second ask moves the answered done into history, it stays cleared there too. - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx index 42091c79de99..162f1705b82f 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx @@ -20,6 +20,7 @@ const mocks = vi.hoisted(() => ({ store: null as null | { getState: () => AppState setState: (state: Partial & { testRuntimeOwner?: string | null }) => void + subscribe: (listener: (state: AppState, previous: AppState) => void) => () => void }, subscribeStatus: vi.fn(), subscribeTranscript: vi.fn(), @@ -30,12 +31,25 @@ const mocks = vi.hoisted(() => ({ vi.mock('@/store', async () => { const { createTestStore } = await import('@/store/slices/store-test-helpers') const useAppStore = createTestStore() - const { setAgentStatus, removeAgentStatus } = useAppStore.getState() + const { setAgentStatus, removeAgentStatus, transactAgentStatuses } = useAppStore.getState() useAppStore.setState({ setAgentStatus: (...args) => { mocks.setAgentStatus(...args) setAgentStatus(...args) }, + // The bridge writes rows through one transaction per tick; each applied row is one write. + transactAgentStatuses: (operation) => + transactAgentStatuses((transaction) => + operation({ + ...transaction, + apply: (update) => { + if (update.kind !== 'providerSession') { + mocks.setAgentStatus(update.paneKey, update.payload) + } + return transaction.apply(update) + } + }) + ), removeAgentStatus: (paneKey) => { mocks.removeAgentStatus(paneKey) removeAgentStatus(paneKey) @@ -164,7 +178,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect(feed().target).toEqual({ kind: 'local' }) expect(mocks.subscribeTranscript).not.toHaveBeenCalled() - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) expect(statuses()).toEqual([ expect.objectContaining({ @@ -188,7 +202,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const updatedAt = Date.now() - 30 * 60 * 1000 - 1 - act(() => feed().emit({ type: 'status', session: summary({ updatedAt }) })) + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt }) })) const entry = statuses()[0] expect(entry).toEqual(expect.objectContaining({ state: 'working', structuredHostOwned: true })) expect(isExplicitAgentStatusFresh(entry, Date.now(), 30 * 60 * 1000)).toBe(true) @@ -197,9 +211,9 @@ describe('StructuredAgentSessionStatusBridge', () => { it('clears host-held evidence when the status stream disconnects', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'status', session: summary() })) + await act(async () => feed().emit({ type: 'status', session: summary() })) expect(statuses()).toHaveLength(1) - act(() => feed().emit({ type: 'end' })) + await act(async () => feed().emit({ type: 'end' })) expect(statuses()).toHaveLength(1) expect(statuses()[0]).not.toHaveProperty('structuredHostOwned') }) @@ -207,15 +221,17 @@ describe('StructuredAgentSessionStatusBridge', () => { it('maps each host status onto the sidebar agent state', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) expect(statuses()).toEqual([expect.objectContaining({ state: 'working' })]) - act(() => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 2 }) })) + await act(async () => + feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 2 }) }) + ) expect(statuses()).toEqual([ expect.objectContaining({ state: 'done', sessionBoundary: false, stateStartedAt: 2 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ status: 'attention', updatedAt: 3 }) }) ) expect(statuses()).toEqual([expect.objectContaining({ state: 'blocked' })]) @@ -225,7 +241,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -262,7 +278,7 @@ describe('StructuredAgentSessionStatusBridge', () => { // An unchanged roster must not rewrite the store. const writes = mocks.setAgentStatus.mock.calls.length - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -282,7 +298,7 @@ describe('StructuredAgentSessionStatusBridge', () => { ) expect(mocks.setAgentStatus.mock.calls.length).toBe(writes) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -299,7 +315,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -317,7 +333,9 @@ describe('StructuredAgentSessionStatusBridge', () => { ).toBe('unverifiable') // A summary without tasks ends the fan-out: children clear with it. - act(() => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 4 }) })) + await act(async () => + feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 4 }) }) + ) expect(statuses()).toEqual([expect.objectContaining({ subagents: undefined })]) }) @@ -327,7 +345,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -346,7 +364,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect.objectContaining({ state: 'working', workingMode: undefined, stateStartedAt: 1 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -364,7 +382,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect.objectContaining({ state: 'working', workingMode: 'monitoring', stateStartedAt: 2 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -388,7 +406,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -404,7 +422,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect.objectContaining({ state: 'working', workingMode: 'monitoring', stateStartedAt: 1 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -430,7 +448,7 @@ describe('StructuredAgentSessionStatusBridge', () => { await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const before = Date.now() - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -458,23 +476,25 @@ describe('StructuredAgentSessionStatusBridge', () => { tab: structuredTab as never, parentIsFresh })[0]?.state - act(() => feed().emit({ type: 'snapshot', sessions: [live] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [live] })) expect(childState()).toBe('unverifiable') parentIsFresh = true expect(childState()).toBe('working') - act(() => feed().emit({ type: 'end' })) + await act(async () => feed().emit({ type: 'end' })) expect(childState()).toBe('unverifiable') await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledTimes(2)) - act(() => feed(1).emit({ type: 'snapshot', sessions: [] })) + await act(async () => feed(1).emit({ type: 'snapshot', sessions: [] })) expect(childState()).toBe('unverifiable') - act(() => feed(1).emit({ type: 'status', session: live })) + await act(async () => feed(1).emit({ type: 'status', session: live })) expect(childState()).toBe('working') parentIsFresh = false expect(childState()).toBe('unverifiable') const writes = mocks.setAgentStatus.mock.calls.length - act(() => feed(1).emit({ type: 'status', session: live })) + await act(async () => feed(1).emit({ type: 'status', session: live })) expect(mocks.setAgentStatus).toHaveBeenCalledTimes(writes) - act(() => feed(1).emit({ type: 'status', session: summary({ backgroundTasks: [] }) })) + await act(async () => + feed(1).emit({ type: 'status', session: summary({ backgroundTasks: [] }) }) + ) expect(childState()).toBeUndefined() }) @@ -482,7 +502,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -505,7 +525,7 @@ describe('StructuredAgentSessionStatusBridge', () => { ]) // The tool line describes live work, so a settled turn that omits it must clear it. - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -527,7 +547,7 @@ describe('StructuredAgentSessionStatusBridge', () => { expect(statuses()[0]?.toolInput).toBeUndefined() // Only the message moves here, so the row updates only if the guard compares it. - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -547,20 +567,20 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary({ status: null })] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: null })] })) expect(mocks.setAgentStatus).not.toHaveBeenCalled() - act(() => feed().emit({ type: 'status', session: summary({ updatedAt: 2 }) })) + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 2 }) })) expect(statuses()).toEqual([expect.objectContaining({ state: 'working' })]) }) it('keeps the status map reference stable for repeated equal summaries', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) const before = mocks.store?.getState().agentStatusByPaneKey - act(() => { + await act(async () => { for (let repeat = 0; repeat < 10; repeat += 1) { feed().emit({ type: 'status', session: summary() }) } @@ -579,7 +599,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: now - 100 })] @@ -593,7 +613,7 @@ describe('StructuredAgentSessionStatusBridge', () => { updatedAt: now - 100 }) ]) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: now - 50 }) }) ) expect(statuses()).toEqual([ @@ -608,8 +628,8 @@ describe('StructuredAgentSessionStatusBridge', () => { it('preserves the working age when host metadata advances during the same turn', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'status', session: summary({ updatedAt: 100 }) })) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 100 }) })) + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 200, providerSession: { ...providerSession, id: 'new-id' } }) @@ -623,15 +643,15 @@ describe('StructuredAgentSessionStatusBridge', () => { it('accepts an authoritative older journal age after a host upgrade reconnect', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'status', session: summary({ updatedAt: 800 }) })) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ updatedAt: 800 }) })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: 900 })] }) ) const paneKey = statuses()[0].paneKey const history = statuses()[0].stateHistory const acknowledged = { [paneKey]: 950 } mocks.store?.setState({ acknowledgedAgentsByPaneKey: acknowledged }) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: 200 })] }) ) expect(statuses()).toEqual([ @@ -641,7 +661,7 @@ describe('StructuredAgentSessionStatusBridge', () => { const calls = mocks.setAgentStatus.mock.calls.length expect(statuses()[0].stateHistory).toBe(history) expect(mocks.store?.getState().acknowledgedAgentsByPaneKey).toBe(acknowledged) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary({ status: 'idle', updatedAt: 200 })] }) ) expect(mocks.store?.getState().agentStatusByPaneKey).toBe(before) @@ -651,10 +671,10 @@ describe('StructuredAgentSessionStatusBridge', () => { it('drops the status and the feed when the last structured tab closes', async () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + await act(async () => feed().emit({ type: 'snapshot', sessions: [summary()] })) expect(statuses()).toHaveLength(1) - act(() => mocks.store?.setState({ unifiedTabsByWorktree: { 'wt-1': [] } })) + await act(async () => mocks.store?.setState({ unifiedTabsByWorktree: { 'wt-1': [] } })) expect(statuses()).toEqual([]) await waitFor(() => expect(mocks.unsubscribe).toHaveBeenCalledOnce()) @@ -667,7 +687,7 @@ describe('StructuredAgentSessionStatusBridge', () => { await act(() => Promise.resolve()) expect(mocks.subscribeStatus).toHaveBeenCalledOnce() - act(() => feed().emit({ type: 'end' })) + await act(async () => feed().emit({ type: 'end' })) await act(() => vi.advanceTimersByTimeAsync(300)) expect(mocks.unsubscribe).toHaveBeenCalledOnce() @@ -707,7 +727,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -717,7 +737,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) ) const rendersWhileStarting = executions.length - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -729,7 +749,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) ) expect(executions).toHaveLength(rendersWhileStarting) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -740,7 +760,7 @@ describe('StructuredAgentSessionStatusBridge', () => { ) expect(executions).toHaveLength(rendersWhileStarting) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -750,7 +770,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) ) expect(executions.at(-1)?.phase).toBe('ready') - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -774,7 +794,7 @@ describe('StructuredAgentSessionStatusBridge', () => { } render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -784,7 +804,7 @@ describe('StructuredAgentSessionStatusBridge', () => { }) ) expect(executions.at(-1)?.childKey).toBe(1) - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ @@ -819,7 +839,7 @@ describe('the main agent fact the bridge writes', () => { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => + await act(async () => feed().emit({ type: 'snapshot', sessions: [ @@ -841,7 +861,7 @@ describe('the main agent fact the bridge writes', () => { ]) // The shell drains: the row settles, the main agent was done all along, so its clock holds. - act(() => + await act(async () => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 2, turnOutcome: 'cancellation' }) diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx index 72669bf936c0..f3c451f8fa25 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx @@ -30,6 +30,7 @@ import { } from '../../../../shared/structured-agent-session-status-started-at' import { getRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner' import { useAppStore } from '@/store' +import type { AgentStatusBatchTransaction } from '@/store/slices/agent-status-contract' import { getActiveRuntimeTarget, type RuntimeClientTarget } from '@/runtime/runtime-rpc-client' import { getStructuredAgentSessionStatusFeed } from '@/runtime/structured-agent-session-status-feed' import { getStructuredAgentSessionTabs, type StructuredTab } from './structured-agent-session-tabs' @@ -102,20 +103,66 @@ function childWorkFor(summary: AgentSessionStatusSummary): { return subagents ? { subagents } : {} } +type StatusProjection = { + tab: StructuredTab + summary: AgentSessionStatusSummary + observation: 'live' | 'unverifiable' +} + +// Rows projected in one tick, applied as one status publication: a host snapshot reaches every +// projection at once, and a store write per row re-runs every reader once per row. +const queuedProjections = new Map() +let projectionFlushQueued = false + function projectStatus( tab: StructuredTab, summary: AgentSessionStatusSummary | null, observation: 'live' | 'unverifiable' ): void { const paneKey = structuredAgentSessionPaneKey(tab.id, tab.entityId) - const store = useAppStore.getState() // No persisted turn yet (or nothing known): the row shows no agent status at all. if (!summary?.status) { + forgetQueuedProjection(paneKey) + const store = useAppStore.getState() if (store.agentStatusByPaneKey?.[paneKey]) { store.removeAgentStatus(paneKey) } return } + queuedProjections.set(paneKey, { tab, summary, observation }) + if (!projectionFlushQueued) { + projectionFlushQueued = true + queueMicrotask(flushQueuedProjections) + } +} + +function forgetQueuedProjection(paneKey: string): void { + queuedProjections.delete(paneKey) +} + +function flushQueuedProjections(): void { + projectionFlushQueued = false + const projections = [...queuedProjections.values()] + queuedProjections.clear() + if (projections.length === 0) { + return + } + useAppStore.getState().transactAgentStatuses((transaction) => { + for (const projection of projections) { + applyStatusProjection(transaction, projection) + } + }) +} + +function applyStatusProjection( + transaction: AgentStatusBatchTransaction, + { tab, summary, observation }: StatusProjection +): void { + const paneKey = structuredAgentSessionPaneKey(tab.id, tab.entityId) + const store = transaction.getState() + if (!summary.status) { + return + } const { children, subagents } = childWorkFor(summary) // Shared with `worktree ps`, so the CLI and this row cannot disagree about one session. const agentStatus = structuredAgentSessionAgentStatus({ @@ -178,11 +225,11 @@ function projectStatus( ) { return } - store.setAgentStatus( + transaction.apply({ paneKey, - desired, - tab.label, - { + payload: desired, + terminalTitle: tab.label, + timing: { updatedAt: summary.updatedAt, // This ordered host feed can correct a legacy publication clock after upgrade. allowOlderTimestamp: true, @@ -199,13 +246,13 @@ function projectStatus( // row held open by child work alone is dated by when this client saw it instead. evidenceObservedAt: isAgentStatusHeldOpenByChildWork(desired) ? Date.now() : summary.updatedAt }, - { tabId: tab.id, worktreeId: tab.worktreeId }, - { + routing: { tabId: tab.id, worktreeId: tab.worktreeId }, + metadata: { ...(summary.providerSession ? { providerSession: summary.providerSession } : {}), terminalResumeEligible: false, ...(summary.hostExecutionOwned ? { structuredHostOwned: true as const } : {}) } - ) + }) } function StructuredAgentSessionStatusProjection({ tab }: { tab: StructuredTab }): null { @@ -221,8 +268,12 @@ function StructuredAgentSessionStatusProjection({ tab }: { tab: StructuredTab }) projectStatus(tab, summary, observation) }, [summary, observation, tab]) useEffect( - () => () => - useAppStore.getState().removeAgentStatus(structuredAgentSessionPaneKey(tab.id, tab.entityId)), + () => () => { + const paneKey = structuredAgentSessionPaneKey(tab.id, tab.entityId) + // A row queued before the unmount must not land after it. + forgetQueuedProjection(paneKey) + useAppStore.getState().removeAgentStatus(paneKey) + }, [tab.entityId, tab.id] ) return null diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx new file mode 100644 index 000000000000..8f2cacba0465 --- /dev/null +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx @@ -0,0 +1,173 @@ +// @vitest-environment happy-dom + +// A host status snapshot reaches every chat's projection at once: its rows must land as one store +// publication, not one per row, or every reader of the status map runs once per row. + +import { act, cleanup, render, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { + AgentSessionStatusEvent, + AgentSessionStatusSummary +} from '../../../../shared/agent-session-wire' +import type { AgentStatusEntry } from '../../../../shared/agent-status-types' +import type { Tab } from '../../../../shared/tab-types' +import type { AppState } from '@/store/types' +import type * as RuntimeRpcClientModule from '@/runtime/runtime-rpc-client' + +const mocks = vi.hoisted(() => ({ + removeAgentStatus: vi.fn(), + setAgentStatus: vi.fn(), + store: null as null | { + getState: () => AppState + setState: (state: Partial & { testRuntimeOwner?: string | null }) => void + subscribe: (listener: (state: AppState, previous: AppState) => void) => () => void + }, + subscribeStatus: vi.fn(), + subscribeTranscript: vi.fn(), + supportsCapability: vi.fn(), + unsubscribe: vi.fn() +})) + +vi.mock('@/store', async () => { + const { createTestStore } = await import('@/store/slices/store-test-helpers') + const useAppStore = createTestStore() + const { setAgentStatus, removeAgentStatus, transactAgentStatuses } = useAppStore.getState() + useAppStore.setState({ + setAgentStatus: (...args) => { + mocks.setAgentStatus(...args) + setAgentStatus(...args) + }, + // The bridge writes rows through one transaction per tick; each applied row is one write. + transactAgentStatuses: (operation) => + transactAgentStatuses((transaction) => + operation({ + ...transaction, + apply: (update) => { + if (update.kind !== 'providerSession') { + mocks.setAgentStatus(update.paneKey, update.payload) + } + return transaction.apply(update) + } + }) + ), + removeAgentStatus: (paneKey) => { + mocks.removeAgentStatus(paneKey) + removeAgentStatus(paneKey) + } + }) + mocks.store = useAppStore + return { useAppStore } +}) + +vi.mock('@/lib/worktree-runtime-owner', () => ({ + getRuntimeEnvironmentIdForWorktree: (state: { testRuntimeOwner?: string | null }) => + state.testRuntimeOwner ?? null +})) + +vi.mock('@/runtime/runtime-rpc-client', async (importOriginal) => ({ + ...(await importOriginal()), + runtimeEnvironmentSupportsCapability: mocks.supportsCapability +})) + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: vi.fn(), + subscribeStructuredAgentSession: mocks.subscribeTranscript, + subscribeStructuredAgentSessionStatus: mocks.subscribeStatus +})) + +import { StructuredAgentSessionStatusBridge } from './StructuredAgentSessionStatusBridge' +import { resetStructuredAgentSessionStatusFeedsForTests } from '@/runtime/structured-agent-session-status-feed' + +const structuredTab = { + id: 'structured-tab-1', + worktreeId: 'wt-1', + groupId: 'group-1', + contentType: 'agent-session', + entityId: 'session-1', + label: 'Codex Chat', + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 0, + isPinned: false, + agentSessionAgent: 'codex' +} satisfies Tab + +const providerSession = { key: 'session_id', id: '01a002e9-9a1c-7d42-a642-e481f64446f1' } as const + +function summary(overrides: Partial = {}): AgentSessionStatusSummary { + return { + sessionId: 'session-1', + workspaceId: 'wt-1', + agent: 'codex', + status: 'working', + hostExecutionOwned: true, + latestPrompt: 'hello', + providerSession, + updatedAt: 1, + ...overrides + } +} + +function statuses(): AgentStatusEntry[] { + return Object.values(mocks.store?.getState().agentStatusByPaneKey ?? {}) +} + +/** The host side of the most recent status subscription. */ +function feed(index = 0): { target: unknown; emit: (event: AgentSessionStatusEvent) => void } { + const call = mocks.subscribeStatus.mock.calls[index] + if (!call) { + throw new Error('status feed not subscribed') + } + return { target: call[0], emit: call[1] as (event: AgentSessionStatusEvent) => void } +} + +describe('a status snapshot of many chats', () => { + beforeEach(() => { + vi.clearAllMocks() + resetStructuredAgentSessionStatusFeedsForTests() + mocks.subscribeStatus.mockResolvedValue({ unsubscribe: mocks.unsubscribe }) + mocks.supportsCapability.mockResolvedValue(true) + mocks.store?.setState({ + agentStatusByPaneKey: {}, + testRuntimeOwner: null, + unifiedTabsByWorktree: { 'wt-1': [structuredTab] } + }) + }) + + afterEach(() => { + cleanup() + resetStructuredAgentSessionStatusFeedsForTests() + }) + + it("applies a snapshot of every listed chat's status as one status publication", async () => { + const tabs = Array.from({ length: 40 }, (_, index) => ({ + ...structuredTab, + id: `structured-tab-${index}`, + entityId: `session-${index}` + })) + mocks.store?.setState({ unifiedTabsByWorktree: { 'wt-1': tabs } }) + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) + // Every publication of the status map re-runs every reader of it, so a row each is a burst. + let publications = 0 + const stop = mocks.store!.subscribe((state, previous) => { + if (state.agentStatusByPaneKey !== previous.agentStatusByPaneKey) { + publications += 1 + } + }) + + await act(async () => + feed().emit({ + type: 'snapshot', + sessions: tabs.map((tab, index) => + summary({ sessionId: tab.entityId, updatedAt: index + 1 }) + ) + }) + ) + stop() + + expect(statuses()).toHaveLength(tabs.length) + expect(publications).toBe(1) + }) +}) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx index c829c9ad3500..52a41b20e9ab 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-clock.test.tsx @@ -102,11 +102,11 @@ function unread(): number { return countActivityUnread(state, ACKNOWLEDGED + 60_000) } -async function connect(): Promise<(event: AgentSessionStatusEvent) => void> { +async function connect(): Promise<(event: AgentSessionStatusEvent) => Promise> { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) const emit: (event: AgentSessionStatusEvent) => void = mocks.subscribeStatus.mock.calls[0]?.[1] - return (event) => act(() => emit(event)) + return (event) => act(async () => emit(event)) } describe("the structured row's state clock", () => { @@ -129,43 +129,43 @@ describe("the structured row's state clock", () => { it('keeps a settled parent read while its subagent writes, until its own next turn ends', async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) + await emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: SETTLED }) acknowledge(ACKNOWLEDGED) expect(unread()).toBe(0) // The child's rows move the evidence clock, and a host may still republish for them. for (const updatedAt of [24_000, 25_000, 26_000]) { - emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt }) }) + await emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt }) }) } expect(row()).toMatchObject({ state: 'done', updatedAt: 26_000, stateStartedAt: SETTLED }) expect(unread()).toBe(0) - emit({ + await emit({ type: 'status', session: summary({ status: 'working', statusStartedAt: 30_000, updatedAt: 30_000 }) }) - emit({ type: 'status', session: summary({ statusStartedAt: 31_000, updatedAt: 31_000 }) }) + await emit({ type: 'status', session: summary({ statusStartedAt: 31_000, updatedAt: 31_000 }) }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: 31_000 }) expect(unread()).toBe(1) }) it("keeps the old dating for an older host's summary, which carries no clock", async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary()] }) + await emit({ type: 'snapshot', sessions: [summary()] }) acknowledge(ACKNOWLEDGED) expect(unread()).toBe(0) - emit({ type: 'status', session: summary({ updatedAt: 26_000 }) }) + await emit({ type: 'status', session: summary({ updatedAt: 26_000 }) }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: 26_000 }) expect(unread()).toBe(1) }) it("dates a subagent's approval at the ask and leaves the parent's completion where it was", async () => { const emit = await connect() - emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) + await emit({ type: 'snapshot', sessions: [summary({ statusStartedAt: SETTLED })] }) acknowledge(ACKNOWLEDGED) - emit({ + await emit({ type: 'status', session: summary({ status: 'attention', statusStartedAt: 27_000, updatedAt: 27_000 }) }) @@ -178,7 +178,10 @@ describe("the structured row's state clock", () => { // Answered: the parent is idle again, still dated by its own last turn. acknowledge(28_000) - emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt: 28_500 }) }) + await emit({ + type: 'status', + session: summary({ statusStartedAt: SETTLED, updatedAt: 28_500 }) + }) expect(row()).toMatchObject({ state: 'done', stateStartedAt: SETTLED }) expect(agentEntryCompletionAt(row())).toBe(SETTLED) expect(unread()).toBe(0) @@ -187,11 +190,11 @@ describe("the structured row's state clock", () => { it('leaves a row child work holds open on its own continuity, and settles it on the parent clock', async () => { const emit = await connect() const child = { id: 'child-1', kind: 'agent', state: 'working' } as const - emit({ + await emit({ type: 'snapshot', sessions: [summary({ status: 'working', statusStartedAt: 10_000, updatedAt: 10_000 })] }) - emit({ + await emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, updatedAt: 24_000, backgroundTasks: [child] }) }) @@ -202,7 +205,7 @@ describe("the structured row's state clock", () => { mainAgent: { state: 'done', stateStartedAt: SETTLED } }) - emit({ + await emit({ type: 'status', session: summary({ statusStartedAt: SETTLED, diff --git a/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx b/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx index ea325e9b155a..c9ef5f394986 100644 --- a/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx +++ b/src/renderer/src/components/native-chat/structured-child-records-switch.test.tsx @@ -235,7 +235,7 @@ describe('the switch: both surfaces read the host child records', () => { } const rendered = render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - act(() => feed()({ type: 'snapshot', sessions: [summary({ children })] })) + await act(async () => feed()({ type: 'snapshot', sessions: [summary({ children })] })) const header = rendered.container.querySelector('[aria-expanded]') if (header) { fireEvent.click(header) @@ -275,9 +275,11 @@ describe('the switch: both surfaces read the host child records', () => { const children = [view('a')] await mountWith(children) const first = row().children - act(() => feed()({ type: 'status', session: summary({ children: structuredClone(children) }) })) + await act(async () => + feed()({ type: 'status', session: summary({ children: structuredClone(children) }) }) + ) expect(row().children).toBe(first) - act(() => + await act(async () => feed()({ type: 'status', session: summary({ children: [{ ...children[0]!, lastMessage: 'Found 3 call sites' }] }) @@ -299,7 +301,7 @@ describe('the switch: both surfaces read the host child records', () => { expect(stripRows(container)).toEqual([live]) // The status stream drops: the host is not heard from, so neither surface vouches for it. - act(() => feed()({ type: 'end' })) + await act(async () => feed()({ type: 'end' })) const lost = { dot: 'No recent update', lead: 'Audit a', trail: 'No update in 2m' } await waitFor(() => expect(sidebarRows()).toEqual([lost])) expect(stripRows(container)).toEqual([lost]) @@ -308,7 +310,7 @@ describe('the switch: both surfaces read the host child records', () => { it('reads a stale parent row the same way on both surfaces', async () => { const { container } = await mountWith([view('a')]) // The host no longer owns the session, and its last word is older than the freshness window. - act(() => + await act(async () => feed()({ type: 'status', session: summary({ @@ -334,7 +336,7 @@ describe('the switch: both surfaces read the host child records', () => { // The child finishes: the host sends no running children, so no roster and no summary rows. rerender() - act(() => feed()({ type: 'status', session: summary({ children: undefined }) })) + await act(async () => feed()({ type: 'status', session: summary({ children: undefined }) })) expect(strip()).toBeNull() expect(sidebarRows()).toEqual([]) From 79ca3cf3a6e0c8da5c15b87f0b809704263f13d2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:23:38 -0700 Subject: [PATCH 058/107] test(native-chat): a 247-chat snapshot reaches the store, the mobile projection and the title batch once --- ...agent-session-status-bridge-burst.test.tsx | 179 +++++++++++++++--- 1 file changed, 151 insertions(+), 28 deletions(-) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx index 8f2cacba0465..521fa93cbc7e 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx @@ -13,10 +13,14 @@ import type { AgentStatusEntry } from '../../../../shared/agent-status-types' import type { Tab } from '../../../../shared/tab-types' import type { AppState } from '@/store/types' import type * as RuntimeRpcClientModule from '@/runtime/runtime-rpc-client' +import type * as AgentStatusProjectionModule from '@/runtime/sync-runtime-graph/agent-status-projection' const mocks = vi.hoisted(() => ({ + mobileStatusProjections: vi.fn(), removeAgentStatus: vi.fn(), setAgentStatus: vi.fn(), + setGeneratedTitle: vi.fn(), + setGeneratedTitles: vi.fn(), store: null as null | { getState: () => AppState setState: (state: Partial & { testRuntimeOwner?: string | null }) => void @@ -31,8 +35,22 @@ const mocks = vi.hoisted(() => ({ vi.mock('@/store', async () => { const { createTestStore } = await import('@/store/slices/store-test-helpers') const useAppStore = createTestStore() - const { setAgentStatus, removeAgentStatus, transactAgentStatuses } = useAppStore.getState() + const { + setAgentStatus, + removeAgentStatus, + transactAgentStatuses, + setGeneratedTabTitleFromAgentPrompt, + setGeneratedTabTitlesFromAgentPrompts + } = useAppStore.getState() useAppStore.setState({ + setGeneratedTabTitleFromAgentPrompt: (...args) => { + mocks.setGeneratedTitle(...args) + setGeneratedTabTitleFromAgentPrompt(...args) + }, + setGeneratedTabTitlesFromAgentPrompts: (updates) => { + mocks.setGeneratedTitles(updates) + setGeneratedTabTitlesFromAgentPrompts(updates) + }, setAgentStatus: (...args) => { mocks.setAgentStatus(...args) setAgentStatus(...args) @@ -59,6 +77,19 @@ vi.mock('@/store', async () => { return { useAppStore } }) +vi.mock('@/runtime/sync-runtime-graph/agent-status-projection', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + buildRuntimeMobileAgentStatusProjection: ( + ...args: Parameters + ) => { + mocks.mobileStatusProjections() + return actual.buildRuntimeMobileAgentStatusProjection(...args) + } + } +}) + vi.mock('@/lib/worktree-runtime-owner', () => ({ getRuntimeEnvironmentIdForWorktree: (state: { testRuntimeOwner?: string | null }) => state.testRuntimeOwner ?? null @@ -77,6 +108,12 @@ vi.mock('@/runtime/structured-agent-session-client', () => ({ import { StructuredAgentSessionStatusBridge } from './StructuredAgentSessionStatusBridge' import { resetStructuredAgentSessionStatusFeedsForTests } from '@/runtime/structured-agent-session-status-feed' +import { + canSkipRuntimeMobileSessionSyncKeyBuild, + getRuntimeMobileSessionSyncKey +} from '@/runtime/sync-runtime-graph/sync-key' +import { getDefaultSettings } from '../../../../shared/constants' +import { structuredAgentSessionPaneKey } from '../../../../shared/structured-agent-session-projection' const structuredTab = { id: 'structured-tab-1', @@ -122,52 +159,138 @@ function feed(index = 0): { target: unknown; emit: (event: AgentSessionStatusEve return { target: call[0], emit: call[1] as (event: AgentSessionStatusEvent) => void } } +// A seeded launch lists a few hundred chats; the host's first snapshot carries all of them. +const CHAT_COUNT = 247 + +const tabs = Array.from({ length: CHAT_COUNT }, (_, index) => ({ + ...structuredTab, + id: `structured-tab-${index}`, + entityId: `session-${index}` +})) + +function chatSummaries(): AgentSessionStatusSummary[] { + return tabs.map((tab, index) => + summary({ + sessionId: tab.entityId, + status: index % 3 === 0 ? 'idle' : 'working', + latestPrompt: `prompt ${index}`, + updatedAt: index + 1 + }) + ) +} + +/** Every store notification, and how many changed the status map. */ +function countPublications(): { all: number; status: number; stop: () => void } { + const counts = { all: 0, status: 0, stop: () => {} } + counts.stop = mocks.store!.subscribe((state, previous) => { + counts.all += 1 + if (state.agentStatusByPaneKey !== previous.agentStatusByPaneKey) { + counts.status += 1 + } + }) + return counts +} + +/** The mobile session sync's store subscriber, which rebuilds the status projection per change. */ +function subscribeMobileSync(): () => void { + let previousKey = getRuntimeMobileSessionSyncKey(mocks.store!.getState()) + return mocks.store!.subscribe((state, previous) => { + if (!canSkipRuntimeMobileSessionSyncKeyBuild(state, previous)) { + previousKey = getRuntimeMobileSessionSyncKey(state, previous, previousKey) + } + }) +} + +async function renderBridge(): Promise { + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) +} + describe('a status snapshot of many chats', () => { beforeEach(() => { vi.clearAllMocks() resetStructuredAgentSessionStatusFeedsForTests() mocks.subscribeStatus.mockResolvedValue({ unsubscribe: mocks.unsubscribe }) mocks.supportsCapability.mockResolvedValue(true) + // The status evidence clock, so rows written on different ticks compare equal. + vi.spyOn(Date, 'now').mockReturnValue(1_000) mocks.store?.setState({ agentStatusByPaneKey: {}, + settings: { ...getDefaultSettings('/tmp'), tabAutoGenerateTitle: true }, testRuntimeOwner: null, - unifiedTabsByWorktree: { 'wt-1': [structuredTab] } + unifiedTabsByWorktree: { 'wt-1': tabs } }) }) afterEach(() => { cleanup() resetStructuredAgentSessionStatusFeedsForTests() + vi.restoreAllMocks() }) - it("applies a snapshot of every listed chat's status as one status publication", async () => { - const tabs = Array.from({ length: 40 }, (_, index) => ({ - ...structuredTab, - id: `structured-tab-${index}`, - entityId: `session-${index}` - })) - mocks.store?.setState({ unifiedTabsByWorktree: { 'wt-1': tabs } }) - render() - await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) - // Every publication of the status map re-runs every reader of it, so a row each is a burst. - let publications = 0 - const stop = mocks.store!.subscribe((state, previous) => { - if (state.agentStatusByPaneKey !== previous.agentStatusByPaneKey) { - publications += 1 - } - }) + it("applies a snapshot of every listed chat's status as one publication", async () => { + await renderBridge() + const stopMobileSync = subscribeMobileSync() + mocks.mobileStatusProjections.mockClear() + // Every publication re-runs every store reader, so a publication per row is the burst. + const publications = countPublications() + await act(async () => feed().emit({ type: 'snapshot', sessions: chatSummaries() })) + publications.stop() + stopMobileSync() + + expect(statuses()).toHaveLength(CHAT_COUNT) + expect(publications.status).toBe(1) + expect(publications.all).toBe(1) + expect(mocks.mobileStatusProjections).toHaveBeenCalledOnce() + // The rows' generated-title requests go out as one batch, not a call per row. + expect(mocks.setGeneratedTitle).not.toHaveBeenCalled() + expect(mocks.setGeneratedTitles).toHaveBeenCalledOnce() + expect(mocks.setGeneratedTitles.mock.calls[0][0]).toHaveLength(CHAT_COUNT) + }) + + it('writes the same rows as a status event per chat', async () => { + await renderBridge() + await act(async () => feed().emit({ type: 'snapshot', sessions: chatSummaries() })) + const fromSnapshot = mocks.store!.getState().agentStatusByPaneKey + + cleanup() + resetStructuredAgentSessionStatusFeedsForTests() + mocks.subscribeStatus.mockClear() + mocks.store?.setState({ agentStatusByPaneKey: {} }) + await renderBridge() + const publications = countPublications() + for (const session of chatSummaries()) { + await act(async () => feed().emit({ type: 'status', session })) + } + publications.stop() + + expect(publications.status).toBe(CHAT_COUNT) + expect(mocks.store!.getState().agentStatusByPaneKey).toEqual(fromSnapshot) + }) + + it('still removes a row, and applies a later single update on the next tick', async () => { + await renderBridge() + await act(async () => feed().emit({ type: 'snapshot', sessions: chatSummaries() })) + const [first, second] = chatSummaries() + const paneKey = (tab: (typeof tabs)[number]): string => + structuredAgentSessionPaneKey(tab.id, tab.entityId) + expect(mocks.store!.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeDefined() + + await act(async () => feed().emit({ type: 'status', session: { ...first, status: null } })) + expect(mocks.removeAgentStatus).toHaveBeenCalledWith(paneKey(tabs[0])) + expect(mocks.store!.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeUndefined() + + const publications = countPublications() await act(async () => - feed().emit({ - type: 'snapshot', - sessions: tabs.map((tab, index) => - summary({ sessionId: tab.entityId, updatedAt: index + 1 }) - ) - }) + feed().emit({ type: 'status', session: { ...second, status: 'idle', updatedAt: 9_999 } }) ) - stop() - - expect(statuses()).toHaveLength(tabs.length) - expect(publications).toBe(1) + publications.stop() + expect(publications.status).toBe(1) + expect(mocks.store!.getState().agentStatusByPaneKey[paneKey(tabs[1])]).toMatchObject({ + state: 'done', + updatedAt: 9_999 + }) + expect(statuses()).toHaveLength(CHAT_COUNT - 1) }) }) From 0786214b740e4eee57710a5c091eeac8e4896238 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:34:03 -0700 Subject: [PATCH 059/107] test(native-chat): listing and Activity tests follow statuses from rows and the once-per-tick bridge update --- ...-agent-session-tab-listing-startup.test.ts | 78 ++++++++++++++----- ...tructuredAgentSessionStatusBridge.test.tsx | 1 - ...agent-session-status-bridge-burst.test.tsx | 35 ++++----- ...x-child-approval-activity-row.unit.test.ts | 11 +-- 4 files changed, 77 insertions(+), 48 deletions(-) diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 2c4b627b7083..098fdc7e8930 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -1,25 +1,51 @@ // The chat tab list a client asks for at startup answers from records and the tab table. It waits -// for no chat's history; each chat's history opens after the answer. +// for no chat's history; each chat's history is read after the answer. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../shared/protocol-version' import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' import { closeTestJournalHostDatabases, - openTestJournalHostDatabase + openTestJournalHostDatabase, + readTestJournalSessionStatus } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import type * as StatusBackfillModule from '../native-chat/agent-session-journal/journal-session-status-backfill' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { createRestTestRig, restTestChat, type RestTestRig } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' -import { latestRestTestStatus } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-observations' +import { + latestRestTestStatus, + restTestOpens +} from '../native-chat/agent-session-wire/structured-agent-session-rest-test-observations' import { AgentSessionStoreTransactions } from './agent-session-store-transactions' import { OrcaRuntimeService } from './orca-runtime' import { RpcDispatcher } from './rpc/dispatcher' import { SESSION_TAB_METHODS } from './rpc/methods/session-tabs' +// Chats whose status cannot be computed from their rows, as an unreadable database page fails it. +const failingDerives = vi.hoisted(() => new Set()) + +vi.mock( + '../native-chat/agent-session-journal/journal-session-status-backfill', + async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + backfillJournalSessionStatus: async ( + ...args: Parameters + ) => { + if (failingDerives.has(args[1])) { + throw new Error('EACCES: permission denied') + } + return actual.backfillJournalSessionStatus(...args) + } + } + } +) + type ListingInternals = { store: { getWorkspaceSession: () => unknown } getClientSettings(): { experimentalStructuredNativeChat: boolean } @@ -47,6 +73,7 @@ beforeEach(async () => { afterEach(async () => { releaseHeld() + failingDerives.clear() // The listing starts the history restore after its answer, without awaiting it; it finishes // before the rig removes its files. await new Promise((resolve) => setImmediate(resolve)) @@ -110,7 +137,7 @@ function restartedRuntime(workspaceSession: unknown = null) { return { runtime, listAll, subscribeAll } } -/** The files an older build left: no chat has stored state, so each is re-derived by its open. */ +/** The files an older build left: no chat has stored state, so each gets it after the answer. */ function forgetStoredChatState(): void { openTestJournalHostDatabase(rig.root).db.exec('DELETE FROM journal_session_state') } @@ -130,7 +157,7 @@ function within(promise: Promise, ms = 2_000): Promise { - it('answers before any chat opens, then opens each one (T1)', async () => { + it('answers before any chat history is read, then gives each chat its status (T1)', async () => { const ids = ['session-1', 'session-2', 'session-3'] for (const sessionId of ids) { await restTestChat(rig, sessionId, { message: sessionId }) @@ -138,22 +165,24 @@ describe('listing chat tabs at startup', () => { await rig.crash() await rig.boot() forgetStoredChatState() - const opens = Promise.withResolvers() - releaseHeld = opens.resolve - let opened = 0 - rig.adapter.historyFilePath.mockImplementation(async () => { - await opens.promise - opened += 1 - return null + const reads = Promise.withResolvers() + releaseHeld = reads.resolve + const restore = rig.host.restoreReadableSessions.bind(rig.host) + vi.spyOn(rig.host, 'restoreReadableSessions').mockImplementation(async (sessionIds) => { + await reads.promise + return restore(sessionIds) }) const { listAll } = restartedRuntime() expect(await within(listAll())).toEqual(ids) - expect(opened).toBe(0) + expect(ids.map((id) => readTestJournalSessionStatus(rig.root, id))).toEqual([null, null, null]) - opens.resolve() - await vi.waitFor(() => ids.forEach((id) => expect(rig.host.hasSession(id)).toBe(true))) - expect(opened).toBe(3) + reads.resolve() + // From the rows alone: a chat with nothing left over is not opened for its status. + await vi.waitFor(() => + ids.forEach((id) => expect(latestRestTestStatus(rig, id)).toMatchObject({ latestPrompt: id })) + ) + expect(ids.map((id) => restTestOpens(rig, id))).toEqual([0, 0, 0]) }) it.each(['listAll', 'subscribeAll'] as const)( @@ -168,16 +197,19 @@ describe('listing chat tabs at startup', () => { forgetStoredChatState() const restore = vi.spyOn(rig.host, 'restoreReadableSessions') const { runtime, listAll, subscribeAll } = restartedRuntime() + const rows = () => + ids.filter((id) => readTestJournalSessionStatus(rig.root, id) !== null).length const seen = () => ({ started: restore.mock.calls.length, - opens: rig.adapter.historyFilePath.mock.calls.length + opens: rig.adapter.historyFilePath.mock.calls.length, + rows: rows() }) const answered = method === 'listAll' ? { ids: await listAll(), atAnswer: seen() } : await subscribeAll(seen) - expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0 } }) - await vi.waitFor(() => ids.forEach((id) => expect(rig.host.hasSession(id)).toBe(true))) + expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0, rows: 0 } }) + await vi.waitFor(() => expect(rows()).toBe(ids.length)) expect(restore).toHaveBeenCalledOnce() runtime.cleanupSubscriptionsForConnection('connection-1') } @@ -296,8 +328,9 @@ describe('listing chat tabs at startup', () => { expect(writes).not.toHaveBeenCalled() }) - it('lists every chat when four fail to open, and the others still get status rows (T5)', async () => { + it('lists every chat when four fail to read, and the others still get status rows (T5)', async () => { // The first four failing: each failure must cost only its own chat, never the chats after it. + // Their rows cannot be read for a status, and their open, which they are left to, fails too. const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5', 'session-6'] const failing = ids.slice(0, 4) for (const sessionId of ids) { @@ -306,6 +339,9 @@ describe('listing chat tabs at startup', () => { await rig.crash() await rig.boot() forgetStoredChatState() + for (const sessionId of failing) { + failingDerives.add(sessionId) + } rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { if (failing.includes(sessionId)) { throw new Error('EACCES: permission denied') @@ -320,7 +356,7 @@ describe('listing chat tabs at startup', () => { await vi.waitFor(() => expect(restore).toHaveBeenCalledOnce()) await restore.mock.results[0]?.value for (const sessionId of ['session-5', 'session-6']) { - expect(rig.host.hasSession(sessionId)).toBe(true) + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ latestPrompt: `asked ${sessionId}` }) diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx index 162f1705b82f..73e86e20acc2 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx @@ -20,7 +20,6 @@ const mocks = vi.hoisted(() => ({ store: null as null | { getState: () => AppState setState: (state: Partial & { testRuntimeOwner?: string | null }) => void - subscribe: (listener: (state: AppState, previous: AppState) => void) => () => void }, subscribeStatus: vi.fn(), subscribeTranscript: vi.fn(), diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx index 521fa93cbc7e..beede025d8d9 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx @@ -11,7 +11,6 @@ import type { } from '../../../../shared/agent-session-wire' import type { AgentStatusEntry } from '../../../../shared/agent-status-types' import type { Tab } from '../../../../shared/tab-types' -import type { AppState } from '@/store/types' import type * as RuntimeRpcClientModule from '@/runtime/runtime-rpc-client' import type * as AgentStatusProjectionModule from '@/runtime/sync-runtime-graph/agent-status-projection' @@ -21,11 +20,6 @@ const mocks = vi.hoisted(() => ({ setAgentStatus: vi.fn(), setGeneratedTitle: vi.fn(), setGeneratedTitles: vi.fn(), - store: null as null | { - getState: () => AppState - setState: (state: Partial & { testRuntimeOwner?: string | null }) => void - subscribe: (listener: (state: AppState, previous: AppState) => void) => () => void - }, subscribeStatus: vi.fn(), subscribeTranscript: vi.fn(), supportsCapability: vi.fn(), @@ -73,7 +67,6 @@ vi.mock('@/store', async () => { removeAgentStatus(paneKey) } }) - mocks.store = useAppStore return { useAppStore } }) @@ -90,9 +83,9 @@ vi.mock('@/runtime/sync-runtime-graph/agent-status-projection', async (importOri } }) +// Every chat runs on the local host. vi.mock('@/lib/worktree-runtime-owner', () => ({ - getRuntimeEnvironmentIdForWorktree: (state: { testRuntimeOwner?: string | null }) => - state.testRuntimeOwner ?? null + getRuntimeEnvironmentIdForWorktree: () => null })) vi.mock('@/runtime/runtime-rpc-client', async (importOriginal) => ({ @@ -107,6 +100,7 @@ vi.mock('@/runtime/structured-agent-session-client', () => ({ })) import { StructuredAgentSessionStatusBridge } from './StructuredAgentSessionStatusBridge' +import { useAppStore } from '@/store' import { resetStructuredAgentSessionStatusFeedsForTests } from '@/runtime/structured-agent-session-status-feed' import { canSkipRuntimeMobileSessionSyncKeyBuild, @@ -147,7 +141,7 @@ function summary(overrides: Partial = {}): AgentSessi } function statuses(): AgentStatusEntry[] { - return Object.values(mocks.store?.getState().agentStatusByPaneKey ?? {}) + return Object.values(useAppStore.getState().agentStatusByPaneKey ?? {}) } /** The host side of the most recent status subscription. */ @@ -182,7 +176,7 @@ function chatSummaries(): AgentSessionStatusSummary[] { /** Every store notification, and how many changed the status map. */ function countPublications(): { all: number; status: number; stop: () => void } { const counts = { all: 0, status: 0, stop: () => {} } - counts.stop = mocks.store!.subscribe((state, previous) => { + counts.stop = useAppStore.subscribe((state, previous) => { counts.all += 1 if (state.agentStatusByPaneKey !== previous.agentStatusByPaneKey) { counts.status += 1 @@ -193,8 +187,8 @@ function countPublications(): { all: number; status: number; stop: () => void } /** The mobile session sync's store subscriber, which rebuilds the status projection per change. */ function subscribeMobileSync(): () => void { - let previousKey = getRuntimeMobileSessionSyncKey(mocks.store!.getState()) - return mocks.store!.subscribe((state, previous) => { + let previousKey = getRuntimeMobileSessionSyncKey(useAppStore.getState()) + return useAppStore.subscribe((state, previous) => { if (!canSkipRuntimeMobileSessionSyncKeyBuild(state, previous)) { previousKey = getRuntimeMobileSessionSyncKey(state, previous, previousKey) } @@ -214,10 +208,9 @@ describe('a status snapshot of many chats', () => { mocks.supportsCapability.mockResolvedValue(true) // The status evidence clock, so rows written on different ticks compare equal. vi.spyOn(Date, 'now').mockReturnValue(1_000) - mocks.store?.setState({ + useAppStore.setState({ agentStatusByPaneKey: {}, settings: { ...getDefaultSettings('/tmp'), tabAutoGenerateTitle: true }, - testRuntimeOwner: null, unifiedTabsByWorktree: { 'wt-1': tabs } }) }) @@ -252,12 +245,12 @@ describe('a status snapshot of many chats', () => { it('writes the same rows as a status event per chat', async () => { await renderBridge() await act(async () => feed().emit({ type: 'snapshot', sessions: chatSummaries() })) - const fromSnapshot = mocks.store!.getState().agentStatusByPaneKey + const fromSnapshot = useAppStore.getState().agentStatusByPaneKey cleanup() resetStructuredAgentSessionStatusFeedsForTests() mocks.subscribeStatus.mockClear() - mocks.store?.setState({ agentStatusByPaneKey: {} }) + useAppStore.setState({ agentStatusByPaneKey: {} }) await renderBridge() const publications = countPublications() for (const session of chatSummaries()) { @@ -266,7 +259,7 @@ describe('a status snapshot of many chats', () => { publications.stop() expect(publications.status).toBe(CHAT_COUNT) - expect(mocks.store!.getState().agentStatusByPaneKey).toEqual(fromSnapshot) + expect(useAppStore.getState().agentStatusByPaneKey).toEqual(fromSnapshot) }) it('still removes a row, and applies a later single update on the next tick', async () => { @@ -275,11 +268,11 @@ describe('a status snapshot of many chats', () => { const [first, second] = chatSummaries() const paneKey = (tab: (typeof tabs)[number]): string => structuredAgentSessionPaneKey(tab.id, tab.entityId) - expect(mocks.store!.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeDefined() + expect(useAppStore.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeDefined() await act(async () => feed().emit({ type: 'status', session: { ...first, status: null } })) expect(mocks.removeAgentStatus).toHaveBeenCalledWith(paneKey(tabs[0])) - expect(mocks.store!.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeUndefined() + expect(useAppStore.getState().agentStatusByPaneKey[paneKey(tabs[0])]).toBeUndefined() const publications = countPublications() await act(async () => @@ -287,7 +280,7 @@ describe('a status snapshot of many chats', () => { ) publications.stop() expect(publications.status).toBe(1) - expect(mocks.store!.getState().agentStatusByPaneKey[paneKey(tabs[1])]).toMatchObject({ + expect(useAppStore.getState().agentStatusByPaneKey[paneKey(tabs[1])]).toMatchObject({ state: 'done', updatedAt: 9_999 }) diff --git a/tests/e2e/codex-child-approval-activity-row.unit.test.ts b/tests/e2e/codex-child-approval-activity-row.unit.test.ts index 27f00644ec1b..abd2ffc7375d 100644 --- a/tests/e2e/codex-child-approval-activity-row.unit.test.ts +++ b/tests/e2e/codex-child-approval-activity-row.unit.test.ts @@ -212,8 +212,9 @@ describe("a Codex subagent's answered approval on the settled parent's Activity const toRenderer: (event: AgentSessionStatusEvent) => void = mocks.subscribeStatus.mock.calls[0]?.[1] let forwarded = 0 - const deliver = (): AgentSessionStatusSummary => { - act(() => { + // Async, so the bridge's once-per-tick store update lands before the row is read. + const deliver = async (): Promise => { + await act(async () => { for (const event of host.events.slice(forwarded)) { toRenderer(event) } @@ -245,7 +246,7 @@ describe("a Codex subagent's answered approval on the settled parent's Activity host.on(CODEX_CHILD, 'turn/started', { turn: { id: 'child-turn' } }) host.on(CODEX_THREAD, 'turn/completed', { turn: { id: 'parent-turn', status: 'completed' } }) await host.drain() - const settled = deliver() + const settled = await deliver() expect(settled).toMatchObject({ status: 'idle', statusStartedAt: expect.any(Number) }) store().setState({ acknowledgedAgentsByPaneKey: { [paneKey()]: host.tick() } }) @@ -259,7 +260,7 @@ describe("a Codex subagent's answered approval on the settled parent's Activity promptKey: 'child-approval' }) await host.drain() - const asked = deliver() + const asked = await deliver() expect(asked).toMatchObject({ status: 'attention' }) expect(asked.statusStartedAt).toBeGreaterThan(settled.statusStartedAt ?? Infinity) // The user reads the ask, then answers it. @@ -268,7 +269,7 @@ describe("a Codex subagent's answered approval on the settled parent's Activity await host.answer(approval ?? '') host.on(CODEX_CHILD, 'turn/completed', { turn: { id: 'child-turn', status: 'completed' } }) await host.drain() - const answered = deliver() + const answered = await deliver() // The host rule under test elsewhere: the answer never re-dates the session's done. expect(answered).toMatchObject({ status: 'idle', statusStartedAt: settled.statusStartedAt }) expect(answered.updatedAt).toBeGreaterThan(asked.updatedAt) From ebdde9d5556b17445200efbfd19ba539760e24ab Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:56:49 -0700 Subject: [PATCH 060/107] fix(native-chat): the startup status pass gives each chat its own task, so short chats never run as one --- .../structured-agent-session-startup-state.ts | 2 + ...ctured-agent-session-status-derive.test.ts | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index ac5f3a25015c..daca71c64456 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -113,6 +113,8 @@ async function deriveMissingStatuses( } const toOpen: string[] = [] for (const sessionId of sessionIds) { + // A task per chat at least: short chats fold in one part, and a pass of them must not be one task. + await yieldTask() if (deps.isDisposed()) { return [] } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts index e385d48596c8..f3143db07608 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts @@ -3,8 +3,11 @@ // with work is still opened, so it is settled; and a send during the pass is not queued behind it. import { afterEach, describe, expect, it, vi } from 'vitest' +import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION } from '../../../shared/agent-session-journal-types' import { closeTestJournalHostDatabases, + insertTestJournalRow, + liveTestJournalRows, openTestJournalHostDatabase, readTestJournalSessionStatus } from '../agent-session-journal/journal-host-database-test-support' @@ -51,6 +54,29 @@ async function startupThroughListing(rig: RestTestRig): Promise { return background } +/** Appends `count` short status items to each chat's history, as a stopped Orca left them. */ +function appendHistory(rig: RestTestRig, sessionIds: readonly string[], count: number): void { + const { db } = openTestJournalHostDatabase(rig.root) + db.exec('BEGIN') + for (const sessionId of sessionIds) { + const tip = liveTestJournalRows(db, sessionId).at(-1)! + for (let index = 0; index < count; index += 1) { + insertTestJournalRow(db, sessionId, { + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + kind: 'item', + epoch: tip.epoch, + seq: tip.seq + 1 + index, + fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, + ts: 1_000 + index, + itemId: `${sessionId}-note-${index}`, + revision: 1, + body: { kind: 'status', text: `note ${index}` } + }) + } + } + db.exec('COMMIT') +} + const seeded = (rig: RestTestRig, sessionId: string) => rig.statusEvents.findIndex( (event) => event.type === 'status' && event.session.sessionId === sessionId @@ -150,4 +176,45 @@ describe('rowless listed chats after an upgrade', () => { expect(after).toBeLessThan(ids.length) expect(derivedCount()).toBe(ids.length) }, 120_000) + + it('gives each short chat its own task, so a send waits for at most one', async () => { + const rig = await newRig() + // Each folds in a single part, so only a yield between chats splits the pass. + const ids = Array.from({ length: 60 }, (_, index) => `session-short-${index}`) + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await restTestChat(rig, 'session-send', { message: 'first' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + appendHistory(rig, ids, 150) + upgradeToEmptyStatusTable(rig) + await rig.boot() + const background = (await startupThroughListing(rig)).filter((id) => id !== 'session-send') + const derivedCount = () => + ids.filter((sessionId) => readTestJournalSessionStatus(rig.root, sessionId) !== null).length + let ticks = 0 + let ticking = true + const tick = (): void => { + if (ticking) { + ticks += 1 + setImmediate(tick) + } + } + setImmediate(tick) + + const pass = rig.host.restoreReadableSessions(background) + await new Promise((resolve) => setImmediate(resolve)) + const before = derivedCount() + const sent = await sendRestTestMessage(rig, 'session-send', 'during the pass') + const after = derivedCount() + await pass + ticking = false + + expect(sent).toMatchObject({ ok: true }) + expect(after - before).toBeLessThanOrEqual(1) + expect(after).toBeLessThan(ids.length) + expect(derivedCount()).toBe(ids.length) + expect(ticks).toBeGreaterThanOrEqual(ids.length) + }, 120_000) }) From 8c23d92488a510f8fafda9086898f60693773942 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:56:51 -0700 Subject: [PATCH 061/107] test(native-chat): batched status rows equal rows written one at a time, across state changes --- ...agent-session-status-bridge-burst.test.tsx | 143 ++++++++++++++---- 1 file changed, 116 insertions(+), 27 deletions(-) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx index beede025d8d9..ae5b0d7742c3 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx +++ b/src/renderer/src/components/native-chat/structured-agent-session-status-bridge-burst.test.tsx @@ -23,7 +23,8 @@ const mocks = vi.hoisted(() => ({ subscribeStatus: vi.fn(), subscribeTranscript: vi.fn(), supportsCapability: vi.fn(), - unsubscribe: vi.fn() + unsubscribe: vi.fn(), + writeEachRowAlone: false })) vi.mock('@/store', async () => { @@ -51,17 +52,46 @@ vi.mock('@/store', async () => { }, // The bridge writes rows through one transaction per tick; each applied row is one write. transactAgentStatuses: (operation) => - transactAgentStatuses((transaction) => - operation({ - ...transaction, - apply: (update) => { - if (update.kind !== 'providerSession') { - mocks.setAgentStatus(update.paneKey, update.payload) - } - return transaction.apply(update) - } - }) - ), + mocks.writeEachRowAlone + ? operation({ + // The bridge before batching: each row written straight to the store, read live. + getState: useAppStore.getState, + apply: (update) => { + const state = useAppStore.getState() + if (update.kind === 'providerSession') { + state.recordAgentProviderSession( + update.paneKey, + update.agent, + update.providerSession, + update.timing, + update.routing, + update.metadata + ) + } else { + state.setAgentStatus( + update.paneKey, + update.payload, + update.terminalTitle, + update.timing, + update.routing, + update.metadata + ) + } + return true + }, + afterCommit: (effect) => effect() + }) + : transactAgentStatuses((transaction) => + operation({ + ...transaction, + apply: (update) => { + if (update.kind !== 'providerSession') { + mocks.setAgentStatus(update.paneKey, update.payload) + } + return transaction.apply(update) + } + }) + ), removeAgentStatus: (paneKey) => { mocks.removeAgentStatus(paneKey) removeAgentStatus(paneKey) @@ -195,6 +225,71 @@ function subscribeMobileSync(): () => void { }) } +/** Every chat in one of five states, shifted each round so each row changes state between rounds. */ +function variedSummaries(round: number): AgentSessionStatusSummary[] { + return tabs.map((tab, index) => { + const kind = (index + round) % 5 + return summary({ + sessionId: tab.entityId, + status: kind === 0 ? 'idle' : kind === 1 ? 'attention' : 'working', + latestPrompt: `prompt ${index} r${round}`, + updatedAt: 100 * round + index + 1, + ...(kind === 0 + ? { turnOutcome: 'interruption' as const, lastAssistantMessage: `said ${index}` } + : {}), + ...(kind === 2 ? { toolName: 'Bash', toolInput: `ls ${index}`, model: 'gpt-x' } : {}), + ...(kind === 3 + ? { + backgroundTasks: [ + { + id: `child-${index}`, + kind: 'agent' as const, + name: 'deep_review', + description: 'Review', + state: 'working' as const, + startedAt: 500 + } + ] + } + : {}), + ...(kind === 4 && round > 0 ? { hostExecutionOwned: undefined } : {}) + }) + }) +} + +/** Three snapshots, then single updates; the status map after each step, and the tab labels. */ +async function snapshotRounds(): Promise<{ rows: unknown[]; labels: unknown[] }> { + await renderBridge() + const rows: unknown[] = [] + for (const round of [0, 1, 2]) { + vi.mocked(Date.now).mockReturnValue(1_000 + round) + await act(async () => feed().emit({ type: 'snapshot', sessions: variedSummaries(round) })) + rows.push(structuredClone(useAppStore.getState().agentStatusByPaneKey)) + } + for (const index of [3, 7, 11]) { + const session = variedSummaries(2)[index]! + await act(async () => + feed().emit({ + type: 'status', + session: { + ...session, + status: 'idle', + turnOutcome: 'interruption', + updatedAt: 50_000 + index + } + }) + ) + } + rows.push(structuredClone(useAppStore.getState().agentStatusByPaneKey)) + const labels = (useAppStore.getState().unifiedTabsByWorktree['wt-1'] ?? []).map((tab) => [ + tab.id, + tab.label, + tab.generatedLabel ?? null + ]) + cleanup() + return { rows, labels } +} + async function renderBridge(): Promise { render() await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) @@ -242,24 +337,18 @@ describe('a status snapshot of many chats', () => { expect(mocks.setGeneratedTitles.mock.calls[0][0]).toHaveLength(CHAT_COUNT) }) - it('writes the same rows as a status event per chat', async () => { - await renderBridge() - await act(async () => feed().emit({ type: 'snapshot', sessions: chatSummaries() })) - const fromSnapshot = useAppStore.getState().agentStatusByPaneKey - - cleanup() + it('writes the same rows and titles as writing each row on its own', async () => { + const tabsBefore = structuredClone(useAppStore.getState().unifiedTabsByWorktree) + const batched = await snapshotRounds() resetStructuredAgentSessionStatusFeedsForTests() mocks.subscribeStatus.mockClear() - useAppStore.setState({ agentStatusByPaneKey: {} }) - await renderBridge() - const publications = countPublications() - for (const session of chatSummaries()) { - await act(async () => feed().emit({ type: 'status', session })) - } - publications.stop() + useAppStore.setState({ agentStatusByPaneKey: {}, unifiedTabsByWorktree: tabsBefore }) + mocks.writeEachRowAlone = true + const perRow = await snapshotRounds() + mocks.writeEachRowAlone = false - expect(publications.status).toBe(CHAT_COUNT) - expect(useAppStore.getState().agentStatusByPaneKey).toEqual(fromSnapshot) + expect(Object.keys(batched.rows[0] ?? {})).toHaveLength(CHAT_COUNT) + expect(batched).toEqual(perRow) }) it('still removes a row, and applies a later single update on the next tick', async () => { From 7bc59c139918ed2d3e9e5d729f3990ff012e08fc Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 04:06:11 -0700 Subject: [PATCH 062/107] test(native-chat): a long chat's status fold spans several tasks even with a task per chat --- .../structured-agent-session-status-derive.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts index f3143db07608..8887d95e7aa6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts @@ -164,17 +164,30 @@ describe('rowless listed chats after an upgrade', () => { const derivedCount = () => ids.filter((sessionId) => readTestJournalSessionStatus(rig.root, sessionId) !== null).length + let ticks = 0 + let ticking = true + const tick = (): void => { + if (ticking) { + ticks += 1 + setImmediate(tick) + } + } + setImmediate(tick) + const pass = rig.host.restoreReadableSessions(background) await new Promise((resolve) => setImmediate(resolve)) const before = derivedCount() const sent = await sendRestTestMessage(rig, 'session-send', 'during the pass') const after = derivedCount() await pass + ticking = false expect(sent).toMatchObject({ ok: true }) expect(after - before).toBeLessThanOrEqual(1) expect(after).toBeLessThan(ids.length) expect(derivedCount()).toBe(ids.length) + // Each long chat folds over several tasks, not one. + expect(ticks).toBeGreaterThanOrEqual(2 * ids.length) }, 120_000) it('gives each short chat its own task, so a send waits for at most one', async () => { From 4d603a2f0c59ccd627d088d06c48f79432042b5f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 04:20:31 -0700 Subject: [PATCH 063/107] fix(native-chat): a listed chat whose startup recovery outlasted its budget is opened unverified, with no second recovery --- docs/reference/agent-status-store.md | 7 +++- .../structured-agent-session-host-types.ts | 2 + .../structured-agent-session-reveal.ts | 9 +++- ...ctured-agent-session-startup-state.test.ts | 42 ++++++++++++++++++- .../structured-agent-session-startup-state.ts | 23 +++++++--- 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 0e1578e8ee22..116b2253b1e6 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -131,8 +131,11 @@ Two rules the ingest must keep: - **Never persist a structured row in the store.** The chat journal database is the durable truth for a structured session. Each chat's status is stored there beside its journal (`journal_session_state`), written in the same - transaction as the journal rows it describes, so it is always current; a - chat last written before that table existed has no row until it is opened. + transaction as the journal rows it describes, so it is always current. A + chat last written before that table existed has no row at first: a listed + chat gets one at startup, computed from its journal rows without opening it + (or from its open, when those rows show work a gone process left); any + other chat gets one when it is opened. At host startup the host republishes a settled chat's row into the store without opening the chat, and settles a chat a gone process left with work first. A structured row in `last-status.json` would diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index 0912e54ce312..22731da7dd3f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -125,6 +125,8 @@ export type StructuredAgentSessionHostDeps = { now?: () => number /** The idle sweep's period and window. Tests drive these; production takes the defaults. */ idleSweep?: { intervalMs?: number; idleMs?: number } + /** How long startup waits on each chat's recovery. Tests shorten it; production takes the default. */ + startupRecoveryBudgetMs?: number /** Whether an orchestration dispatch still owns this session's worker; absent answers no. */ hasOpenDispatch?: (record: AgentSessionRecord) => boolean /** Where every failure the host carries on past is reported. Required: a host without one would diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 74cbb175d940..608cb938e4a8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -115,8 +115,13 @@ export function createStructuredAgentSessionHostRestore( canSettle, seedStatus, resolveRecovery: readRestore.resolveRecovery, - restoreListed: (records) => - restoreStructuredAgentSessionsOnRestart({ ...readRestore, records }), + restoreListed: (records, resolveListedRecovery) => + restoreStructuredAgentSessionsOnRestart({ + ...readRestore, + records, + resolveRecovery: resolveListedRecovery + }), + recoveryBudgetMs: deps.startupRecoveryBudgetMs, serialize: rest.serialize, hasSession: rest.hasSession, isDisposed: rest.isDisposed diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index d0d1e5f7eeac..928d1a6138af 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -623,7 +623,7 @@ describe('a recovery that never answers', () => { await state.settleOwedSessions([]) expect(resolveRecovery).toHaveBeenCalledWith('session-stuck') - expect(restoreListed).toHaveBeenCalledWith([]) + expect(restoreListed).toHaveBeenCalledWith([], expect.any(Function)) expect(stuck.lease.handoffStage).toBe('recovering') expect(logger.warn).toHaveBeenCalledWith('a chat recovery outlasted startup; left unverified', { scope: 'startup-recovery-timeout', @@ -632,6 +632,46 @@ describe('a recovery that never answers', () => { }) }) +describe('a listed chat whose recovery never answers', () => { + it('is opened unverified, with no second recovery beside the first, and the rest still settle', async () => { + const rig = await newRig() + await crashMidSend(rig, 'session-stuck') + await crashMidSend(rig, 'session-closed', false) + const lease = rig.store.getRecord('session-stuck')!.lease + await rig.crash() + // Its provider process was up when Orca died; the check of it answers once, then never again. + await writeOlderBuildLease(rig.root, 'session-stuck', { ...lease }) + let answered = false + let hung = 0 + rig.probeOwner.mockImplementation(async (record) => { + if (record.sessionId !== 'session-stuck') { + return { outcome: 'pid-absent' } + } + if (!answered) { + answered = true + return { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + } + hung += 1 + return new Promise(() => {}) + }) + await rig.boot({ startupRecoveryBudgetMs: 50 }) + expect(listedIds(rig)).toEqual(['session-stuck']) + + const done = await Promise.race([ + startup(rig, listedIds(rig)).then(() => 'done'), + new Promise((resolve) => setTimeout(() => resolve('still waiting'), 5_000)) + ]) + + expect(done).toBe('done') + expect(hung).toBe(1) + expect(rig.store.getRecord('session-stuck')!.lease.handoffStage).toBe('recovering') + expect(opened(rig, ['session-closed'])).toEqual(['session-closed']) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle' + }) + }, 20_000) +}) + describe('a stored status no settle here can clear (R2A-4)', () => { it('is refused by the shared closed-chat settle, which opens nothing for it', async () => { const opens: string[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index daca71c64456..91dfd47148c8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -50,8 +50,12 @@ export type StructuredAgentSessionStartupStateDeps = { /** Resolves a `recovering` lease; never throws (a failure is reported and left to the next * attach or send). */ resolveRecovery: (sessionId: string) => Promise - /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish). */ - restoreListed: (records: AgentSessionRecord[]) => Promise + /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish), resolving + * recovery through `resolveRecovery`. */ + restoreListed: ( + records: AgentSessionRecord[], + resolveRecovery: (sessionId: string) => Promise + ) => Promise /** Tests shorten it; production takes the default. */ recoveryBudgetMs?: number serialize: (sessionId: string, task: () => Promise) => Promise @@ -226,7 +230,7 @@ async function settleOwedSessions( if (database.readOnly) { return } - await resolveRecoveringLeases(deps) + const outlasted = await resolveRecoveringLeases(deps) const listedOrder = new Map(listedIds.map((sessionId, index) => [sessionId, index])) const listed: AgentSessionRecord[] = [] const others: AgentSessionRecord[] = [] @@ -246,7 +250,11 @@ async function settleOwedSessions( (left, right) => (listedOrder.get(left.sessionId) ?? 0) - (listedOrder.get(right.sessionId) ?? 0) ) - await deps.restoreListed(listed) + // A recovery that outlasted its budget is still running: a second one beside it could hang the + // same way, so its chat is opened unverified and the next attach or send resolves it. + await deps.restoreListed(listed, (sessionId) => + outlasted.has(sessionId) ? Promise.resolve(true) : deps.resolveRecovery(sessionId) + ) // A listed chat the worker left closed (its tab closed meanwhile) is settled like any other. others.push(...listed.filter((record) => !deps.hasSession(record.sessionId))) for (const record of others) { @@ -300,21 +308,26 @@ function dropUnreachableStatus( /** Every lease a crash left `recovering`, listed or not: a provider process that outlived the crash * is stopped and its death recorded, so the settle below reads one verdict per turn. */ -async function resolveRecoveringLeases(deps: StructuredAgentSessionStartupStateDeps) { +async function resolveRecoveringLeases( + deps: StructuredAgentSessionStartupStateDeps +): Promise> { const recovering = deps.openDeps.store .listRecords() .filter((record) => record.lease.handoffStage === 'recovering') const budgetMs = deps.recoveryBudgetMs ?? RECOVERY_BUDGET_MS + const outlasted = new Set() await forEachWithConcurrency(recovering, RECOVERY_CONCURRENCY, async ({ sessionId }) => { if ( (await withTimeout(deps.resolveRecovery(sessionId), budgetMs, null)) === null ) { + outlasted.add(sessionId) deps.openDeps.logger.warn('a chat recovery outlasted startup; left unverified', { scope: 'startup-recovery-timeout', sessionId }) } }) + return outlasted } /** A chat nothing holds open: settled and closed, never indexed, so it gets no status row. */ From 84805b6cecee33e3e63a4810c7bf5e0b310e8e1f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:00:59 -0700 Subject: [PATCH 064/107] perf(native-chat): the startup status pass writes its rows a slice at a time, in one transaction per slice --- .../journal-session-status-backfill.test.ts | 72 ++++++++- .../journal-session-status-backfill.ts | 94 +++++++---- .../structured-agent-session-startup-state.ts | 69 +-------- ...red-agent-session-startup-status-derive.ts | 122 +++++++++++++++ ...ctured-agent-session-status-derive.test.ts | 146 ++++++++++++++---- ...-agent-session-tab-listing-startup.test.ts | 6 +- 6 files changed, 380 insertions(+), 129 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index 9b6da6149cc8..1428c7c3af1e 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -9,11 +9,17 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session- import { createTrackedJournalOpener, insertTestJournalRowJson, + publishTestJournalEpoch, openTestJournalHostDatabase, readTestJournalSessionStatus } from './journal-host-database-test-support' import * as JournalOpen from './journal-open' -import { backfillJournalSessionStatus } from './journal-session-status-backfill' +import { + backfillJournalSessionStatus, + foldJournalSessionStatus, + writeJournalSessionStatuses, + type FoldedJournalSessionStatus +} from './journal-session-status-backfill' import { CORPUS_FENCE, JOURNAL_SESSION_STATE_CASES, @@ -191,4 +197,68 @@ describe('a row from the rows alone', () => { // A row already there is never rewritten. expect(await backfillJournalSessionStatus(database(), 'moving')).toBeNull() }) + + it('writes a whole batch in one transaction, each row the row an open writes', async () => { + const byOpen = new Map() + for (const name of JOURNAL_SESSION_STATE_CASES) { + await JOURNAL_SESSION_STATE_CORPUS[name](await open(name)) + } + await journals.closeAll() + for (const name of JOURNAL_SESSION_STATE_CASES) { + dropRow(name) + ;(await open(name)).backfillSessionStatus() + byOpen.set(name, readTestJournalSessionStatus(root, name)) + await journals.closeAll() + dropRow(name) + } + const folded: FoldedJournalSessionStatus[] = [] + for (const name of JOURNAL_SESSION_STATE_CASES) { + folded.push((await foldJournalSessionStatus(database(), name))!) + } + const transaction = vi.spyOn(database(), 'transaction') + + const written = writeJournalSessionStatuses(database(), folded) + + expect(transaction).toHaveBeenCalledOnce() + expect(written.map(({ sessionId }) => sessionId)).toEqual([...JOURNAL_SESSION_STATE_CASES]) + for (const name of JOURNAL_SESSION_STATE_CASES) { + expect(readTestJournalSessionStatus(root, name)).toEqual(byOpen.get(name)) + } + }) + + it('skips only the chats in a batch that moved, got a row or a new history since their fold', async () => { + const ids = ['steady', 'appended', 'rowed', 'replaced'] + const journalsById = new Map() + for (const sessionId of ids) { + const journal = await open(sessionId) + await JOURNAL_SESSION_STATE_CORPUS.settled(journal) + journalsById.set(sessionId, journal) + dropRow(sessionId) + } + const folded: FoldedJournalSessionStatus[] = [] + for (const sessionId of ids) { + folded.push((await foldJournalSessionStatus(database(), sessionId))!) + } + // Between the fold and the batch's write: a send lands, an open writes the row, a rebuild + // starts a new history. + await journalsById + .get('appended')! + .appendItem( + { provider: 'orca', clientMessageId: 'late' }, + { kind: 'status', text: 'late' }, + { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } + ) + dropRow('appended') + journalsById.get('rowed')!.backfillSessionStatus() + const rowedBefore = readTestJournalSessionStatus(root, 'rowed') + publishTestJournalEpoch(database().db, 'replaced', 'epoch-replaced-later') + + const written = writeJournalSessionStatuses(database(), folded) + + expect(written.map(({ sessionId }) => sessionId)).toEqual(['steady']) + expect(readTestJournalSessionStatus(root, 'steady')).toEqual(folded[0]!.status) + expect(readTestJournalSessionStatus(root, 'appended')).toBeNull() + expect(readTestJournalSessionStatus(root, 'rowed')).toEqual(rowedBefore) + expect(readTestJournalSessionStatus(root, 'replaced')).toBeNull() + }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index 05a0f9b6bba4..a0b91486932e 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -3,8 +3,8 @@ // Version 5 creates the status table empty, so after an upgrade every chat has no row until // something writes it. Rather than open each chat (its lease, settle and conversation), its rows are // folded a bounded part per task, as a replay folds them, then the row is derived and written in one -// short transaction, only while the chat is still where the fold read it. The startup pass and the -// background copy both write missing rows through this one function. +// short transaction, only while the chat is still where the fold read it. The startup pass folds a +// slice of chats and writes their rows in one transaction; the background copy writes one at a time. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { JournalHostDatabase } from './journal-host-database' @@ -45,12 +45,43 @@ export function* charBoundedBatches( } } -/** The chat's row: its rows folded a part per task, as a replay folds them, then the row derived - * and written in one short transaction, only while the chat is still where the fold read it. Null - * when something wrote the row or the chat first, the chat has no epoch in this database (it is - * still in a per-chat file), the database or the chat's rows are a newer build's, or `signal` - * aborted, which is checked before each part. */ +/** A chat's status folded from its rows, and where the fold read them. */ +export type FoldedJournalSessionStatus = { + sessionId: string + epoch: string + tip: number + load: JournalLoad + status: JournalSessionStatus +} + +type FoldOptions = { + batchRows?: number + batchChars?: number + /** Ends each part's task: the next macrotask by default; a background job paces here. */ + yieldTask?: () => Promise + /** Quit: the fold stops within one part and nothing is written. */ + signal?: AbortSignal +} + +/** The chat's row and status: folded, then written alone. Null when something wrote the row or + * moved the chat first, or as `foldJournalSessionStatus` says. */ export async function backfillJournalSessionStatus( + database: JournalHostDatabase, + sessionId: string, + options: FoldOptions = {} +): Promise<{ load: JournalLoad; status: JournalSessionStatus } | null> { + const folded = await foldJournalSessionStatus(database, sessionId, options) + if (!folded || options.signal?.aborted) { + return null + } + return writeJournalSessionStatuses(database, [folded]).length > 0 ? folded : null +} + +/** The chat's rows folded a part per task, as a replay folds them, and its status derived. Null when + * the chat already has a row, has no epoch in this database (it is still in a per-chat file), the + * database or the chat's rows are a newer build's, or `signal` aborted, which is checked before each + * part. Writes nothing. */ +export async function foldJournalSessionStatus( database: JournalHostDatabase, sessionId: string, { @@ -58,15 +89,8 @@ export async function backfillJournalSessionStatus( batchChars = IMPORT_BATCH_CHARS, yieldTask = () => yieldToEventLoop(), signal - }: { - batchRows?: number - batchChars?: number - /** Ends each part's task: the next macrotask by default; a background job paces here. */ - yieldTask?: () => Promise - /** Quit: the fold stops within one part and nothing is written. */ - signal?: AbortSignal - } = {} -): Promise<{ load: JournalLoad; status: JournalSessionStatus } | null> { + }: FoldOptions = {} +): Promise { if (database.readOnly) { return null } @@ -112,16 +136,30 @@ export async function backfillJournalSessionStatus( if (load.readOnly) { return null } - return database.transaction((db) => { - if ( - hasJournalSessionStatus(db, sessionId) || - readJournalSessionEpoch(db, sessionId) !== epoch || - readJournalTip(db, sessionId, epoch) !== tip - ) { - return null - } - const status = deriveJournalSessionStatus(load.state, { settlesRosters: !load.corrupt }) - writeJournalSessionStatus(db, sessionId, status) - return { load, status } - }) + const status = deriveJournalSessionStatus(load.state, { settlesRosters: !load.corrupt }) + return { sessionId, epoch, tip, load, status } +} + +/** Every folded chat's row in ONE transaction, each only while it still has no row and its rows are + * where its fold read them. Answers the chats written; a chat skipped is left to its open. */ +export function writeJournalSessionStatuses( + database: JournalHostDatabase, + folded: readonly FoldedJournalSessionStatus[] +): FoldedJournalSessionStatus[] { + if (folded.length === 0) { + return [] + } + return database.transaction((db) => + folded.filter(({ sessionId, epoch, tip, status }) => { + if ( + hasJournalSessionStatus(db, sessionId) || + readJournalSessionEpoch(db, sessionId) !== epoch || + readJournalTip(db, sessionId, epoch) !== tip + ) { + return false + } + writeJournalSessionStatus(db, sessionId, status) + return true + }) + ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 91dfd47148c8..bde2dddb714b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -23,12 +23,12 @@ import { readJournalSessionStatuses, readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' -import { backfillJournalSessionStatus } from '../agent-session-journal/journal-session-status-backfill' import { openStructuredAgentSessionConversationJournal, type StructuredAgentSessionConversationOpenDeps } from './structured-agent-session-conversation-open' import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' +import { deriveMissingStatuses } from './structured-agent-session-startup-status-derive' // Record-only (a probe and at most a process stop each), so a few at once. const RECOVERY_CONCURRENCY = 4 @@ -100,73 +100,6 @@ export function createStructuredAgentSessionStartupState( } } -async function deriveMissingStatuses( - deps: StructuredAgentSessionStartupStateDeps, - sessionIds: readonly string[] -): Promise { - if (deps.openDeps.journalDatabase.readOnly) { - return [...sessionIds] - } - // Quit stops a fold within one part, and nothing more is written. - const quit = new AbortController() - const yieldTask = async () => { - await yieldToEventLoop() - if (deps.isDisposed()) { - quit.abort() - } - } - const toOpen: string[] = [] - for (const sessionId of sessionIds) { - // A task per chat at least: short chats fold in one part, and a pass of them must not be one task. - await yieldTask() - if (deps.isDisposed()) { - return [] - } - const record = deps.openDeps.store.getRecord(sessionId) - const derived = - record && - deps.canSettle(record) && - !deps.hasSession(sessionId) && - (await deriveFromRows(deps, record, { yieldTask, signal: quit.signal })) - if (!derived) { - toOpen.push(sessionId) - } - } - return toOpen -} - -/** Seeds the chat from a row derived from its rows; false leaves it to its open. */ -async function deriveFromRows( - deps: StructuredAgentSessionStartupStateDeps, - record: AgentSessionRecord, - options: { yieldTask: () => Promise; signal: AbortSignal } -): Promise { - let derived: Awaited> - try { - derived = await backfillJournalSessionStatus( - deps.openDeps.journalDatabase, - record.sessionId, - options - ) - } catch (error) { - deps.openDeps.logger.warn('deriving a chat status from its rows failed', { - scope: 'startup-status-derive', - sessionId: record.sessionId, - error - }) - return false - } - // Work a gone process left is settled, and a corrupt history rebuilt, by the chat's open. - if (!derived || derived.load.corrupt || isUnsettledJournalSessionStatus(derived.status)) { - return false - } - deps.seedStatus(record, { - projected: derived.status.summary, - lastActivityAt: derived.status.lastActivityAt - }) - return true -} - function seedStoredStatuses( deps: StructuredAgentSessionStartupStateDeps, listedIds: readonly string[] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts new file mode 100644 index 000000000000..0f827e858eef --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts @@ -0,0 +1,122 @@ +// The background pass's first step after an upgrade: listed chats with history here and no status +// row get their rows from their rows alone, without an open. Chats fold one per task; their rows are +// written a slice at a time, in one transaction per slice, so the pass appends few WAL pages. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + foldJournalSessionStatus, + writeJournalSessionStatuses, + type FoldedJournalSessionStatus +} from '../agent-session-journal/journal-session-status-backfill' +import { isUnsettledJournalSessionStatus } from '../agent-session-journal/journal-session-state' +import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' + +// A write per slice, not per chat: each per-chat commit rewrites the same table and index pages. +const SLICE_CHATS = 16 +const SLICE_MS = 50 + +type Folded = FoldedJournalSessionStatus & { record: AgentSessionRecord } + +/** Answers the chats still to open, in the order given: the rest, and any whose rows show work a + * gone process left or owe a rebuild, which their open settles or rebuilds. */ +export async function deriveMissingStatuses( + deps: StructuredAgentSessionStartupStateDeps, + sessionIds: readonly string[] +): Promise { + if (deps.openDeps.journalDatabase.readOnly) { + return [...sessionIds] + } + // Quit stops a fold within one part, and nothing more is written. + const quit = new AbortController() + const yieldTask = async () => { + await yieldToEventLoop() + if (deps.isDisposed()) { + quit.abort() + } + } + const toOpen: string[] = [] + let slice: Folded[] = [] + let sliceStart = 0 + for (const sessionId of sessionIds) { + // A task per chat at least: short chats fold in one part, and a pass of them must not be one task. + await yieldTask() + if (deps.isDisposed()) { + return [] + } + const record = deps.openDeps.store.getRecord(sessionId) + const folded = + record && deps.canSettle(record) && !deps.hasSession(sessionId) + ? await foldFromRows(deps, record, { yieldTask, signal: quit.signal }) + : null + if (!folded) { + toOpen.push(sessionId) + continue + } + if (slice.length === 0) { + sliceStart = performance.now() + } + slice.push(folded) + if (slice.length >= SLICE_CHATS || performance.now() - sliceStart >= SLICE_MS) { + toOpen.push(...writeSlice(deps, slice)) + slice = [] + } + } + if (deps.isDisposed()) { + return [] + } + toOpen.push(...writeSlice(deps, slice)) + return toOpen +} + +/** A chat's settled status from its rows; null leaves it to its open. */ +async function foldFromRows( + deps: StructuredAgentSessionStartupStateDeps, + record: AgentSessionRecord, + options: { yieldTask: () => Promise; signal: AbortSignal } +): Promise { + try { + const folded = await foldJournalSessionStatus( + deps.openDeps.journalDatabase, + record.sessionId, + options + ) + // Work a gone process left is settled, and a corrupt history rebuilt, by the chat's open. + if (!folded || folded.load.corrupt || isUnsettledJournalSessionStatus(folded.status)) { + return null + } + return { ...folded, record } + } catch (error) { + deps.openDeps.logger.warn('deriving a chat status from its rows failed', { + scope: 'startup-status-derive', + sessionId: record.sessionId, + error + }) + return null + } +} + +/** Writes the slice's rows in one transaction and seeds each written chat; answers the chats left to + * their open (one a send or an open moved meanwhile, or all of them if the write failed). */ +function writeSlice(deps: StructuredAgentSessionStartupStateDeps, slice: Folded[]): string[] { + let written: FoldedJournalSessionStatus[] + try { + written = writeJournalSessionStatuses(deps.openDeps.journalDatabase, slice) + } catch (error) { + deps.openDeps.logger.warn('writing chat statuses derived from their rows failed', { + scope: 'startup-status-derive', + error + }) + return slice.map(({ sessionId }) => sessionId) + } + const writtenIds = new Set(written.map(({ sessionId }) => sessionId)) + for (const folded of slice) { + if (writtenIds.has(folded.sessionId)) { + deps.seedStatus(folded.record, { + projected: folded.status.summary, + lastActivityAt: folded.status.lastActivityAt + }) + } + } + return slice.flatMap(({ sessionId }) => (writtenIds.has(sessionId) ? [] : [sessionId])) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts index 8887d95e7aa6..363be6f36585 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts @@ -1,9 +1,13 @@ // The first boot after upgrading: listed chats with history and no status row get their rows from -// their rows alone, one at a time in tab order, with no conversation opened; a chat a crash left -// with work is still opened, so it is settled; and a send during the pass is not queued behind it. +// their rows alone, folded one at a time in the order given and written a slice at a time, with no +// conversation opened; a chat a crash left with work is still opened, so it is settled; and a send +// during the pass is not queued behind it. +import { statSync } from 'node:fs' +import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION } from '../../../shared/agent-session-journal-types' +import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' import { closeTestJournalHostDatabases, insertTestJournalRow, @@ -22,9 +26,27 @@ import { restTestOpens } from './structured-agent-session-rest-test-observations' +// The chats folded so far, whether or not their slice's rows are written yet. +const folds = vi.hoisted(() => ({ done: new Set() })) + +vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + foldJournalSessionStatus: async ( + ...args: Parameters + ) => { + const folded = await actual.foldJournalSessionStatus(...args) + folds.done.add(args[1]) + return folded + } + } +}) + const rigs: RestTestRig[] = [] afterEach(async () => { + folds.done.clear() for (const rig of rigs.splice(0)) { await rig.dispose() } @@ -107,38 +129,102 @@ describe('rowless listed chats after an upgrade', () => { expect(order).toEqual(order.toSorted((a, b) => a - b)) }) - it('still opens, and so settles, a rowless chat a crash left mid-turn', async () => { + it('shows no status for a rowless chat until it has a row, and never seeds an unfinished one from its rows', async () => { const rig = await newRig() await restTestChat(rig, 'session-fine', { message: 'done' }) - await restTestChat(rig, 'session-crashed', { message: 'asked' }) - const [{ providerIdentity }] = await Promise.all( - rig.adapter.dispatch.mock.results.slice(-1).map((result) => result.value) - ) - await rig.host - .collaboratorsForTests() - .sessions.get('session-crashed')! - .journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { - fence: rig.store.getRecord('session-crashed')!.lease.runtimeFence, - turnScope: { kind: 'thread' } - } - ) + // A turn still running, and an approval still waiting, when Orca died. + const unfinished = ['session-running', 'session-prompt'] + for (const sessionId of unfinished) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value + const journal = rig.host.collaboratorsForTests().sessions.get(sessionId)!.journal + const options = { + fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, + turnScope: { kind: 'thread' as const } + } + await (sessionId === 'session-running' + ? journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + options + ) + : journal.appendItem( + { ...providerIdentity, ordinal: 50 }, + { + kind: 'approval', + title: 'Approve?', + detail: null, + options: [], + resolution: { + state: 'pending', + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null + } + }, + options + )) + } await rig.crash() upgradeToEmptyStatusTable(rig) await rig.boot() + const background = await startupThroughListing(rig) + // No row yet: no status at all, rather than a guess. + for (const sessionId of ['session-fine', ...unfinished]) { + expect(latestRestTestStatus(rig, sessionId)).toBeUndefined() + } + // The status stream's length when each chat first opened. + const openedAt = new Map() + rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { + if (!openedAt.has(sessionId)) { + openedAt.set(sessionId, rig.statusEvents.length) + } + return null + }) - await rig.host.restoreReadableSessions(await startupThroughListing(rig)) + await rig.host.restoreReadableSessions(background) expect(restTestOpens(rig, 'session-fine')).toBe(0) - expect(restTestOpens(rig, 'session-crashed')).toBeGreaterThan(0) - expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ - lifecycle: 'idle' - }) + expect(latestRestTestStatus(rig, 'session-fine')).toMatchObject({ status: 'idle' }) + for (const sessionId of unfinished) { + // Its first status comes from its open, which settles it: never a row derived beforehand. + expect(openedAt.get(sessionId)).toBeDefined() + expect(seeded(rig, sessionId)).toBeGreaterThanOrEqual(openedAt.get(sessionId)!) + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) + expect(latestRestTestStatus(rig, sessionId)?.status).not.toBe('attention') + expect(latestRestTestStatus(rig, sessionId)?.status).not.toBe('working') + } }) - it('accepts a send during the pass with at most one chat derived ahead of it', async () => { + it('appends few WAL pages for a pass of 247 chats: their rows are written a slice at a time', async () => { + const rig = await newRig() + const ids = Array.from({ length: 247 }, (_, index) => `session-page-${index}`) + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + const background = await startupThroughListing(rig) + const { db } = openTestJournalHostDatabase(rig.root) + // An empty WAL, so its size after the pass is what the pass appended. + db.pragma('wal_checkpoint(TRUNCATE)') + const pageSize = Number(db.pragma('page_size', { simple: true })) + + await rig.host.restoreReadableSessions(background) + + const frames = + (statSync(join(rig.root, 'agent-session-journal.db-wal')).size - 32) / (pageSize + 24) + expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) + expect( + ids.map((sessionId) => restTestOpens(rig, sessionId)).every((opens) => opens === 0) + ).toBe(true) + // A commit per chat appends about 2.5 pages each, over 600 for this pass. + expect(frames).toBeLessThanOrEqual(150) + }, 120_000) + + it('accepts a send during the pass with at most one chat folded, and one slice written, ahead of it', async () => { const rig = await newRig() const ids = Array.from({ length: 6 }, (_, index) => `session-long-${index}`) for (const sessionId of ids) { @@ -163,6 +249,7 @@ describe('rowless listed chats after an upgrade', () => { const background = (await startupThroughListing(rig)).filter((id) => id !== 'session-send') const derivedCount = () => ids.filter((sessionId) => readTestJournalSessionStatus(rig.root, sessionId) !== null).length + const foldedCount = () => ids.filter((sessionId) => folds.done.has(sessionId)).length let ticks = 0 let ticking = true @@ -176,9 +263,9 @@ describe('rowless listed chats after an upgrade', () => { const pass = rig.host.restoreReadableSessions(background) await new Promise((resolve) => setImmediate(resolve)) - const before = derivedCount() + const before = foldedCount() const sent = await sendRestTestMessage(rig, 'session-send', 'during the pass') - const after = derivedCount() + const after = foldedCount() await pass ticking = false @@ -190,7 +277,7 @@ describe('rowless listed chats after an upgrade', () => { expect(ticks).toBeGreaterThanOrEqual(2 * ids.length) }, 120_000) - it('gives each short chat its own task, so a send waits for at most one', async () => { + it('gives each short chat its own task, so a send waits for at most one fold and one slice write', async () => { const rig = await newRig() // Each folds in a single part, so only a yield between chats splits the pass. const ids = Array.from({ length: 60 }, (_, index) => `session-short-${index}`) @@ -206,6 +293,7 @@ describe('rowless listed chats after an upgrade', () => { const background = (await startupThroughListing(rig)).filter((id) => id !== 'session-send') const derivedCount = () => ids.filter((sessionId) => readTestJournalSessionStatus(rig.root, sessionId) !== null).length + const foldedCount = () => ids.filter((sessionId) => folds.done.has(sessionId)).length let ticks = 0 let ticking = true const tick = (): void => { @@ -218,9 +306,9 @@ describe('rowless listed chats after an upgrade', () => { const pass = rig.host.restoreReadableSessions(background) await new Promise((resolve) => setImmediate(resolve)) - const before = derivedCount() + const before = foldedCount() const sent = await sendRestTestMessage(rig, 'session-send', 'during the pass') - const after = derivedCount() + const after = foldedCount() await pass ticking = false diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 098fdc7e8930..a4a1dd0fd2a1 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -34,13 +34,13 @@ vi.mock( const actual = await importOriginal() return { ...actual, - backfillJournalSessionStatus: async ( - ...args: Parameters + foldJournalSessionStatus: async ( + ...args: Parameters ) => { if (failingDerives.has(args[1])) { throw new Error('EACCES: permission denied') } - return actual.backfillJournalSessionStatus(...args) + return actual.foldJournalSessionStatus(...args) } } } From 92b791a7a81dd7fef16a52dce296fec9a08dfc32 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:01:00 -0700 Subject: [PATCH 065/107] perf(native-chat): the startup status pass fills the chats on screen first --- ...ructured-agent-session-restoration.test.ts | 65 ++++++++++++++++++- ...ed-structured-agent-session-restoration.ts | 31 +++++++++ .../structured-agent-session-startup-step.ts | 8 ++- 3 files changed, 101 insertions(+), 3 deletions(-) diff --git a/src/main/runtime/saved-structured-agent-session-restoration.test.ts b/src/main/runtime/saved-structured-agent-session-restoration.test.ts index 8a28f0fd8d06..f57e6928d81f 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.test.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from 'vitest' import type { Tab } from '../../shared/tab-types' import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' -import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' +import { + collectSavedStructuredAgentSessionIds, + orderOnScreenStructuredAgentSessionsFirst +} from './saved-structured-agent-session-restoration' +import { runStructuredAgentSessionStartupStep } from './structured-agent-session-startup-step' function tab(input: Partial & Pick): Tab { const { id, ...overrides } = input @@ -82,4 +86,63 @@ describe('saved structured session restoration targets', () => { expect(collectSavedStructuredAgentSessionIds(saved)).toEqual(['session-codex']) }) + + it("fills the chats on screen first: the active worktree's active tab, then every other worktree's", async () => { + const tabs = { + 'workspace-1': [ + tab({ id: 'a-1', entityId: 'session-a-1' }), + tab({ id: 'a-2', entityId: 'session-a-2' }) + ], + 'workspace-2': [ + tab({ id: 'b-1', worktreeId: 'workspace-2', entityId: 'session-b-1' }), + tab({ + id: 'b-2', + worktreeId: 'workspace-2', + entityId: 'session-b-2', + agentSessionAgent: 'claude' + }), + tab({ + id: 'b-remote', + worktreeId: 'workspace-2', + entityId: 'session-b-remote', + executionHostId: 'ssh:build' + }) + ] + } + const saved: WorkspaceSessionState = { + ...session([], null), + activeWorktreeId: 'workspace-2', + unifiedTabs: tabs, + activeTabIdByWorktree: { 'workspace-1': 'a-2', 'workspace-2': 'b-2' } + } + // The pass's order: the host's tab order, which knows nothing of what is on screen. + const listed = ['session-a-1', 'session-a-2', 'session-b-1', 'session-b-2', 'session-b-remote'] + + expect(orderOnScreenStructuredAgentSessionsFirst(listed, saved)).toEqual([ + 'session-b-2', + 'session-a-2', + 'session-a-1', + 'session-b-1', + 'session-b-remote' + ]) + // Only reorders the chats it is given. + expect(orderOnScreenStructuredAgentSessionsFirst(['session-a-1'], saved)).toEqual([ + 'session-a-1' + ]) + expect(orderOnScreenStructuredAgentSessionsFirst(listed, null)).toEqual(listed) + + const host = { + reconcileRestartLeases: async () => undefined, + getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: listed }), + seedStoredStatuses: (ids: readonly string[]) => [...ids], + settleOwedSessions: async () => undefined + } + expect(await runStructuredAgentSessionStartupStep(host, saved, () => undefined)).toEqual([ + 'session-b-2', + 'session-a-2', + 'session-a-1', + 'session-b-1', + 'session-b-remote' + ]) + }) }) diff --git a/src/main/runtime/saved-structured-agent-session-restoration.ts b/src/main/runtime/saved-structured-agent-session-restoration.ts index 71d617de81a0..321575e5eeb3 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.ts @@ -41,3 +41,34 @@ export function collectSavedStructuredAgentSessionIds( } return selected } + +/** + * The chats a launch shows first, those whose tab is active in its worktree, the active worktree's + * first, ahead of the rest in their given order. Any agent: this orders, it restores nothing. + */ +export function orderOnScreenStructuredAgentSessionsFirst( + sessionIds: readonly string[], + session: WorkspaceSessionState | null +): string[] { + const tabsById = new Map( + Object.values(session?.unifiedTabs ?? {}) + .flat() + .map((tab) => [tab.id, tab]) + ) + const activeByWorktree = session?.activeTabIdByWorktree ?? {} + const activeWorktree = session?.activeWorktreeId ?? null + const activeTabIds = [ + ...(activeWorktree ? [activeByWorktree[activeWorktree]] : []), + ...Object.values(activeByWorktree) + ] + const listed = new Set(sessionIds) + const first = new Set() + for (const tabId of activeTabIds) { + const tab = typeof tabId === 'string' ? tabsById.get(tabId) : undefined + const local = !tab?.executionHostId || tab.executionHostId === LOCAL_EXECUTION_HOST_ID + if (tab?.contentType === 'agent-session' && local && listed.has(tab.entityId)) { + first.add(tab.entityId) + } + } + return [...first, ...sessionIds.filter((sessionId) => !first.has(sessionId))] +} diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index f8b95b44f438..401ef4a7a45a 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -5,7 +5,10 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' -import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' +import { + collectSavedStructuredAgentSessionIds, + orderOnScreenStructuredAgentSessionsFirst +} from './saved-structured-agent-session-restoration' import { ensureStructuredAgentSessionHostUnlessRefused } from './structured-agent-session-host-refusal' import type { StartupStepOutcome, @@ -39,7 +42,8 @@ export async function runStructuredAgentSessionStartupStep( const background = host.seedStoredStatuses(listedIds) // Not awaited here: the tab list and paint never wait on it; chat commands do. onSettling(host.settleOwedSessions(listedIds)) - return background + // The rows the window shows at launch fill first. + return orderOnScreenStructuredAgentSessionsFirst(background, savedSession) } /** The chats with a tab, for the startup step and the tab restore alike: the host's persisted tab From a4b1950f03936b37b964a2981c08ac5cdfe8c05e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:26:40 -0700 Subject: [PATCH 066/107] fix(native-chat): the startup pass finds the chats on screen from the saved tab groups, not a map that only names terminals --- ...ructured-agent-session-restoration.test.ts | 24 ++++- ...ed-structured-agent-session-restoration.ts | 31 +++--- ...structured-chat-startup-fill-order.test.ts | 99 +++++++++++++++++++ 3 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts diff --git a/src/main/runtime/saved-structured-agent-session-restoration.test.ts b/src/main/runtime/saved-structured-agent-session-restoration.test.ts index f57e6928d81f..f51757e8841a 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.test.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.test.ts @@ -87,7 +87,7 @@ describe('saved structured session restoration targets', () => { expect(collectSavedStructuredAgentSessionIds(saved)).toEqual(['session-codex']) }) - it("fills the chats on screen first: the active worktree's active tab, then every other worktree's", async () => { + it("fills the chats on screen first: the active worktree's visible tabs, then every other worktree's", async () => { const tabs = { 'workspace-1': [ tab({ id: 'a-1', entityId: 'session-a-1' }), @@ -109,20 +109,36 @@ describe('saved structured session restoration targets', () => { }) ] } + const group = (id: string, worktreeId: string, activeTabId: string) => ({ + id, + worktreeId, + activeTabId, + tabOrder: [activeTabId] + }) + // As the renderer saves it: each group's active tab is what is on screen, the focused group's + // first; `activeTabIdByWorktree` names a terminal whatever is shown. const saved: WorkspaceSessionState = { ...session([], null), activeWorktreeId: 'workspace-2', unifiedTabs: tabs, - activeTabIdByWorktree: { 'workspace-1': 'a-2', 'workspace-2': 'b-2' } + tabGroups: { + 'workspace-1': [group('g-a', 'workspace-1', 'a-2')], + 'workspace-2': [ + group('g-b-left', 'workspace-2', 'b-1'), + group('g-b-right', 'workspace-2', 'b-2') + ] + }, + activeGroupIdByWorktree: { 'workspace-1': 'g-a', 'workspace-2': 'g-b-right' }, + activeTabIdByWorktree: { 'workspace-1': 'terminal-a', 'workspace-2': 'terminal-b' } } // The pass's order: the host's tab order, which knows nothing of what is on screen. const listed = ['session-a-1', 'session-a-2', 'session-b-1', 'session-b-2', 'session-b-remote'] expect(orderOnScreenStructuredAgentSessionsFirst(listed, saved)).toEqual([ 'session-b-2', + 'session-b-1', 'session-a-2', 'session-a-1', - 'session-b-1', 'session-b-remote' ]) // Only reorders the chats it is given. @@ -139,9 +155,9 @@ describe('saved structured session restoration targets', () => { } expect(await runStructuredAgentSessionStartupStep(host, saved, () => undefined)).toEqual([ 'session-b-2', + 'session-b-1', 'session-a-2', 'session-a-1', - 'session-b-1', 'session-b-remote' ]) }) diff --git a/src/main/runtime/saved-structured-agent-session-restoration.ts b/src/main/runtime/saved-structured-agent-session-restoration.ts index 321575e5eeb3..d778692e474e 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.ts @@ -43,8 +43,10 @@ export function collectSavedStructuredAgentSessionIds( } /** - * The chats a launch shows first, those whose tab is active in its worktree, the active worktree's - * first, ahead of the rest in their given order. Any agent: this orders, it restores nothing. + * The chats a launch shows first, each tab group's active tab (the active worktree's first, and in + * each worktree its focused group first), ahead of the rest in their given order. Read from the + * groups because `activeTabIdByWorktree` only ever holds a terminal's tab. Any agent: this orders, + * it restores nothing. */ export function orderOnScreenStructuredAgentSessionsFirst( sessionIds: readonly string[], @@ -55,19 +57,26 @@ export function orderOnScreenStructuredAgentSessionsFirst( .flat() .map((tab) => [tab.id, tab]) ) - const activeByWorktree = session?.activeTabIdByWorktree ?? {} + const groupsByWorktree = session?.tabGroups ?? {} const activeWorktree = session?.activeWorktreeId ?? null - const activeTabIds = [ - ...(activeWorktree ? [activeByWorktree[activeWorktree]] : []), - ...Object.values(activeByWorktree) + const worktrees = [ + ...(activeWorktree && groupsByWorktree[activeWorktree] ? [activeWorktree] : []), + ...Object.keys(groupsByWorktree).filter((worktreeId) => worktreeId !== activeWorktree) ] const listed = new Set(sessionIds) const first = new Set() - for (const tabId of activeTabIds) { - const tab = typeof tabId === 'string' ? tabsById.get(tabId) : undefined - const local = !tab?.executionHostId || tab.executionHostId === LOCAL_EXECUTION_HOST_ID - if (tab?.contentType === 'agent-session' && local && listed.has(tab.entityId)) { - first.add(tab.entityId) + for (const worktreeId of worktrees) { + const groups = groupsByWorktree[worktreeId] ?? [] + const focused = session?.activeGroupIdByWorktree?.[worktreeId] + for (const group of [ + ...groups.filter((candidate) => candidate.id === focused), + ...groups.filter((candidate) => candidate.id !== focused) + ]) { + const tab = group.activeTabId ? tabsById.get(group.activeTabId) : undefined + const local = !tab?.executionHostId || tab.executionHostId === LOCAL_EXECUTION_HOST_ID + if (tab?.contentType === 'agent-session' && local && listed.has(tab.entityId)) { + first.add(tab.entityId) + } } } return [...first, ...sessionIds.filter((sessionId) => !first.has(sessionId))] diff --git a/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts b/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts new file mode 100644 index 000000000000..e4a0fb0b3719 --- /dev/null +++ b/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts @@ -0,0 +1,99 @@ +// The startup status pass fills the chats on screen first, read from the workspace session the +// renderer really saves: a chat the user is looking at is its tab group's active tab, while +// `activeTabIdByWorktree` keeps naming a terminal. + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as AgentStatusModule from '@/lib/agent-status' +import { buildWorkspaceSessionPayload } from '@/lib/workspace-session' +import type { Tab } from '../../../../shared/tab-types' +import { orderOnScreenStructuredAgentSessionsFirst } from '../../../../main/runtime/saved-structured-agent-session-restoration' +import { createTabsSliceMockApi } from './tabs-slice-test-harness' +import { createTestStore } from './store-test-helpers' +import { buildActiveSurfacePatch } from './tabs/tabs-surface' + +vi.mock('sonner', () => ({ toast: { info: vi.fn(), success: vi.fn(), error: vi.fn() } })) +vi.mock('@/lib/agent-status', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, detectAgentStatusFromTitle: vi.fn().mockReturnValue(null) } +}) +createTabsSliceMockApi() + +const WORKTREE = 'repo1::/tmp/feature' +const GROUP = 'group-1' + +function tab(id: string, contentType: Tab['contentType'], sortOrder: number): Tab { + return { + id, + entityId: contentType === 'agent-session' ? `session-${id.slice(-1)}` : id, + groupId: GROUP, + worktreeId: WORKTREE, + contentType, + label: id, + customLabel: null, + color: null, + sortOrder, + createdAt: sortOrder, + ...(contentType === 'agent-session' ? { agentSessionAgent: 'codex' as const } : {}) + } +} + +describe('the chats on screen at launch', () => { + let store: ReturnType + + beforeEach(() => { + store = createTestStore() + const tabs = [ + tab('terminal-1', 'terminal', 0), + tab('structured-session-1', 'agent-session', 1), + tab('structured-session-2', 'agent-session', 2), + tab('structured-session-3', 'agent-session', 3) + ] + store.setState({ + activeWorktreeId: WORKTREE, + unifiedTabsByWorktree: { [WORKTREE]: tabs }, + groupsByWorktree: { + [WORKTREE]: [ + { + id: GROUP, + worktreeId: WORKTREE, + activeTabId: 'terminal-1', + tabOrder: tabs.map(({ id }) => id) + } + ] + }, + activeGroupIdByWorktree: { [WORKTREE]: GROUP }, + tabsByWorktree: { + [WORKTREE]: [ + { + id: 'terminal-1', + ptyId: 'pty-1', + worktreeId: WORKTREE, + title: 'Terminal 1', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 0 + } + ] + }, + ptyIdsByTabId: { 'terminal-1': ['pty-1'] }, + activeTabIdByWorktree: { [WORKTREE]: 'terminal-1' } + }) + }) + + it('come first in the pass, as the saved session records the chat the user is looking at', () => { + // The user looks at the third chat; then the worktree's surface is applied, as a focus does. + store.getState().activateTab('structured-session-3') + store.setState(buildActiveSurfacePatch(store.getState(), WORKTREE)) + + const saved = buildWorkspaceSessionPayload(store.getState()) + const listed = ['session-1', 'session-2', 'session-3'] + + expect(saved.activeTabIdByWorktree?.[WORKTREE]).toBe('terminal-1') + expect(orderOnScreenStructuredAgentSessionsFirst(listed, saved)).toEqual([ + 'session-3', + 'session-1', + 'session-2' + ]) + }) +}) From ff1408d091b9bb26ae3b8fb4794e611c17f5adf5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:26:42 -0700 Subject: [PATCH 067/107] fix(native-chat): the startup pass stores unfinished chats' rows unshown and seeds no closed chat A rowless chat whose rows show work a gone process left, or whose history is corrupt, has its row written again (as before the slice batching), never shown from rows: only its open, which settles or rebuilds it, publishes its status. If its tab closes before that open, the stored row makes the next boot select and settle it. A chat whose tab closed, or that something opened, between its fold and its slice's write gets its row written but no seeded sidebar row. --- .../structured-agent-session-reveal.ts | 1 + ...ctured-agent-session-startup-state.test.ts | 2 + .../structured-agent-session-startup-state.ts | 9 +- ...red-agent-session-startup-status-derive.ts | 41 ++++++--- ...ctured-agent-session-status-derive.test.ts | 89 ++++++++++++++++++- 5 files changed, 125 insertions(+), 17 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 608cb938e4a8..df9fca150dbf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -124,6 +124,7 @@ export function createStructuredAgentSessionHostRestore( recoveryBudgetMs: deps.startupRecoveryBudgetMs, serialize: rest.serialize, hasSession: rest.hasSession, + isListed: readRestore.isListed, isDisposed: rest.isDisposed }) return { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 928d1a6138af..bdab9bba5ffe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -616,6 +616,7 @@ describe('a recovery that never answers', () => { restoreListed, serialize: (_sessionId, task) => task(), hasSession: () => false, + isListed: () => true, isDisposed: () => false, recoveryBudgetMs: 20 }) @@ -700,6 +701,7 @@ describe('a stored status no settle here can clear (R2A-4)', () => { restoreListed: vi.fn(async () => undefined), serialize: (_sessionId, task) => task(), hasSession: () => false, + isListed: () => true, isDisposed: () => false }) const record = agentSessionRecordFixture() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index bde2dddb714b..d7ba67bad4ef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -60,6 +60,8 @@ export type StructuredAgentSessionStartupStateDeps = { recoveryBudgetMs?: number serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean + /** Whether the chat still has its tab. */ + isListed: (sessionId: string) => boolean isDisposed: () => boolean } @@ -78,9 +80,10 @@ export type StructuredAgentSessionStartupState = { settleClosedChat: (record: AgentSessionRecord) => Promise /** * The background pass's first step: each listed chat with history here and no status row gets - * its row from its rows alone, opening nothing, one chat at a time in the order given. Answers - * the chats still to open: the rest, and any whose rows show work a gone process left or owe a - * rebuild, which their open settles or rebuilds. + * its row from its rows alone, opening nothing, folded one chat at a time in the order given (the + * startup step puts the chats on screen first) and written a slice at a time. A row showing work + * a gone process left, or a corrupt history, is stored but never shown. Answers the chats still + * to open, which their open settles, rebuilds or gives a row; see `deriveMissingStatuses`. */ deriveMissingStatuses: (sessionIds: readonly string[]) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts index 0f827e858eef..f6f0503c0ff4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts @@ -16,10 +16,12 @@ import type { StructuredAgentSessionStartupStateDeps } from './structured-agent- const SLICE_CHATS = 16 const SLICE_MS = 50 -type Folded = FoldedJournalSessionStatus & { record: AgentSessionRecord } +/** `shown` false: a row stored so the next boot selects the chat, but never displayed from rows. */ +type Folded = FoldedJournalSessionStatus & { record: AgentSessionRecord; shown: boolean } -/** Answers the chats still to open, in the order given: the rest, and any whose rows show work a - * gone process left or owe a rebuild, which their open settles or rebuilds. */ +/** Answers the chats still to open: the chats it could not fold, and those whose rows show work a + * gone process left or owe a rebuild, which their open settles or rebuilds, in the order given; + * then, after each slice's write, the chats of that slice the write had to skip. */ export async function deriveMissingStatuses( deps: StructuredAgentSessionStartupStateDeps, sessionIds: readonly string[] @@ -49,8 +51,10 @@ export async function deriveMissingStatuses( record && deps.canSettle(record) && !deps.hasSession(sessionId) ? await foldFromRows(deps, record, { yieldTask, signal: quit.signal }) : null - if (!folded) { + if (!folded || !folded.shown) { toOpen.push(sessionId) + } + if (!folded) { continue } if (slice.length === 0) { @@ -69,7 +73,9 @@ export async function deriveMissingStatuses( return toOpen } -/** A chat's settled status from its rows; null leaves it to its open. */ +/** A chat's status from its rows; null leaves it to its open. One with work a gone process left, + * or a corrupt history, is still written so startup selects it until its open settles it, but it + * is never shown: only its open, which settles or rebuilds it, publishes its status. */ async function foldFromRows( deps: StructuredAgentSessionStartupStateDeps, record: AgentSessionRecord, @@ -81,11 +87,11 @@ async function foldFromRows( record.sessionId, options ) - // Work a gone process left is settled, and a corrupt history rebuilt, by the chat's open. - if (!folded || folded.load.corrupt || isUnsettledJournalSessionStatus(folded.status)) { + if (!folded) { return null } - return { ...folded, record } + const shown = !folded.load.corrupt && !isUnsettledJournalSessionStatus(folded.status) + return { ...folded, record, shown } } catch (error) { deps.openDeps.logger.warn('deriving a chat status from its rows failed', { scope: 'startup-status-derive', @@ -96,8 +102,9 @@ async function foldFromRows( } } -/** Writes the slice's rows in one transaction and seeds each written chat; answers the chats left to - * their open (one a send or an open moved meanwhile, or all of them if the write failed). */ +/** Writes the slice's rows in one transaction and seeds each written chat still listed and closed; + * answers the shown chats left to their open (one a send or an open moved meanwhile, or all of them + * if the write failed). */ function writeSlice(deps: StructuredAgentSessionStartupStateDeps, slice: Folded[]): string[] { let written: FoldedJournalSessionStatus[] try { @@ -107,16 +114,24 @@ function writeSlice(deps: StructuredAgentSessionStartupStateDeps, slice: Folded[ scope: 'startup-status-derive', error }) - return slice.map(({ sessionId }) => sessionId) + return slice.flatMap(({ sessionId, shown }) => (shown ? [sessionId] : [])) } const writtenIds = new Set(written.map(({ sessionId }) => sessionId)) for (const folded of slice) { - if (writtenIds.has(folded.sessionId)) { + // A tab closed since the fold gets no sidebar row; one opened since publishes its own. + if ( + folded.shown && + writtenIds.has(folded.sessionId) && + deps.isListed(folded.sessionId) && + !deps.hasSession(folded.sessionId) + ) { deps.seedStatus(folded.record, { projected: folded.status.summary, lastActivityAt: folded.status.lastActivityAt }) } } - return slice.flatMap(({ sessionId }) => (writtenIds.has(sessionId) ? [] : [sessionId])) + return slice.flatMap(({ sessionId, shown }) => + shown && !writtenIds.has(sessionId) ? [sessionId] : [] + ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts index 363be6f36585..b03ecce6e5e2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts @@ -21,13 +21,21 @@ import { sendRestTestMessage, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' import { latestRestTestStatus, restTestOpens } from './structured-agent-session-rest-test-observations' // The chats folded so far, whether or not their slice's rows are written yet. -const folds = vi.hoisted(() => ({ done: new Set() })) +const folds = vi.hoisted(() => { + const hooks: { + done: Set + /** Runs as each fold ends, before its slice is written. */ + after: ((sessionId: string) => Promise) | null + } = { done: new Set(), after: null } + return hooks +}) vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { const actual = await importOriginal() @@ -38,6 +46,7 @@ vi.mock('../agent-session-journal/journal-session-status-backfill', async (impor ) => { const folded = await actual.foldJournalSessionStatus(...args) folds.done.add(args[1]) + await folds.after?.(args[1]) return folded } } @@ -47,6 +56,7 @@ const rigs: RestTestRig[] = [] afterEach(async () => { folds.done.clear() + folds.after = null for (const rig of rigs.splice(0)) { await rig.dispose() } @@ -76,6 +86,20 @@ async function startupThroughListing(rig: RestTestRig): Promise { return background } +/** A chat whose turn was still running when Orca died. */ +async function crashMidTurn(rig: RestTestRig, sessionId: string): Promise { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value + await rig.host + .collaboratorsForTests() + .sessions.get(sessionId)! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } + ) +} + /** Appends `count` short status items to each chat's history, as a stopped Orca left them. */ function appendHistory(rig: RestTestRig, sessionIds: readonly string[], count: number): void { const { db } = openTestJournalHostDatabase(rig.root) @@ -318,4 +342,67 @@ describe('rowless listed chats after an upgrade', () => { expect(derivedCount()).toBe(ids.length) expect(ticks).toBeGreaterThanOrEqual(ids.length) }, 120_000) + + it('writes the row of a chat whose tab closes mid-pass, but gives it no sidebar row', async () => { + const rig = await newRig() + for (const sessionId of ['session-kept', 'session-closing']) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + const background = await startupThroughListing(rig) + // The tab closes after the chat's fold, before its slice is written. + folds.after = async (sessionId) => { + if (sessionId === 'session-closing') { + await rig.host.setSessionTabVisibility(sessionId, false) + } + } + + await rig.host.restoreReadableSessions(background) + + expect(readTestJournalSessionStatus(rig.root, 'session-closing')).toMatchObject({ + lifecycle: 'idle' + }) + expect(seeded(rig, 'session-closing')).toBe(-1) + expect(restTestOpens(rig, 'session-closing')).toBe(0) + expect(latestRestTestStatus(rig, 'session-kept')).toMatchObject({ status: 'idle' }) + }) + + it("stores an unfinished chat's row unshown, so a tab closed before its open is still settled on the next boot", async () => { + const rig = await newRig() + await crashMidTurn(rig, 'session-crashed') + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + const background = await startupThroughListing(rig) + // Closed after its fold, so the pass's open of it never happens. + folds.after = async (sessionId) => { + if (sessionId === 'session-crashed') { + await rig.host.setSessionTabVisibility(sessionId, false) + } + } + + await rig.host.restoreReadableSessions(background) + + expect(restTestOpens(rig, 'session-crashed')).toBe(0) + expect(seeded(rig, 'session-crashed')).toBe(-1) + expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ + lifecycle: 'running' + }) + expect(readUnsettledJournalSessionIds(openTestJournalHostDatabase(rig.root).db)).toEqual([ + 'session-crashed' + ]) + + // The next boot selects it from its row and settles it, tab or no tab. + folds.after = null + await rig.crash() + await rig.boot() + await startupThroughListing(rig) + expect(restTestOpens(rig, 'session-crashed')).toBeGreaterThan(0) + expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ + lifecycle: 'idle' + }) + }) }) From f950b363fbf8cca3c803b3f92dcc00143ebb8cdd Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:35:32 -0700 Subject: [PATCH 068/107] refactor(native-chat): the on-screen chat order lives in shared, so the renderer's own session test can read it --- ...ructured-agent-session-restoration.test.ts | 6 +-- ...ed-structured-agent-session-restoration.ts | 40 ----------------- .../structured-agent-session-startup-step.ts | 6 +-- ...structured-chat-startup-fill-order.test.ts | 2 +- ...ved-on-screen-structured-agent-sessions.ts | 44 +++++++++++++++++++ 5 files changed, 49 insertions(+), 49 deletions(-) create mode 100644 src/shared/saved-on-screen-structured-agent-sessions.ts diff --git a/src/main/runtime/saved-structured-agent-session-restoration.test.ts b/src/main/runtime/saved-structured-agent-session-restoration.test.ts index f51757e8841a..d677be52eed8 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.test.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.test.ts @@ -1,10 +1,8 @@ import { describe, expect, it } from 'vitest' import type { Tab } from '../../shared/tab-types' import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' -import { - collectSavedStructuredAgentSessionIds, - orderOnScreenStructuredAgentSessionsFirst -} from './saved-structured-agent-session-restoration' +import { orderOnScreenStructuredAgentSessionsFirst } from '../../shared/saved-on-screen-structured-agent-sessions' +import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { runStructuredAgentSessionStartupStep } from './structured-agent-session-startup-step' function tab(input: Partial & Pick): Tab { diff --git a/src/main/runtime/saved-structured-agent-session-restoration.ts b/src/main/runtime/saved-structured-agent-session-restoration.ts index d778692e474e..71d617de81a0 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.ts @@ -41,43 +41,3 @@ export function collectSavedStructuredAgentSessionIds( } return selected } - -/** - * The chats a launch shows first, each tab group's active tab (the active worktree's first, and in - * each worktree its focused group first), ahead of the rest in their given order. Read from the - * groups because `activeTabIdByWorktree` only ever holds a terminal's tab. Any agent: this orders, - * it restores nothing. - */ -export function orderOnScreenStructuredAgentSessionsFirst( - sessionIds: readonly string[], - session: WorkspaceSessionState | null -): string[] { - const tabsById = new Map( - Object.values(session?.unifiedTabs ?? {}) - .flat() - .map((tab) => [tab.id, tab]) - ) - const groupsByWorktree = session?.tabGroups ?? {} - const activeWorktree = session?.activeWorktreeId ?? null - const worktrees = [ - ...(activeWorktree && groupsByWorktree[activeWorktree] ? [activeWorktree] : []), - ...Object.keys(groupsByWorktree).filter((worktreeId) => worktreeId !== activeWorktree) - ] - const listed = new Set(sessionIds) - const first = new Set() - for (const worktreeId of worktrees) { - const groups = groupsByWorktree[worktreeId] ?? [] - const focused = session?.activeGroupIdByWorktree?.[worktreeId] - for (const group of [ - ...groups.filter((candidate) => candidate.id === focused), - ...groups.filter((candidate) => candidate.id !== focused) - ]) { - const tab = group.activeTabId ? tabsById.get(group.activeTabId) : undefined - const local = !tab?.executionHostId || tab.executionHostId === LOCAL_EXECUTION_HOST_ID - if (tab?.contentType === 'agent-session' && local && listed.has(tab.entityId)) { - first.add(tab.entityId) - } - } - } - return [...first, ...sessionIds.filter((sessionId) => !first.has(sessionId))] -} diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index 401ef4a7a45a..11221f2da156 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -5,10 +5,8 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' -import { - collectSavedStructuredAgentSessionIds, - orderOnScreenStructuredAgentSessionsFirst -} from './saved-structured-agent-session-restoration' +import { orderOnScreenStructuredAgentSessionsFirst } from '../../shared/saved-on-screen-structured-agent-sessions' +import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { ensureStructuredAgentSessionHostUnlessRefused } from './structured-agent-session-host-refusal' import type { StartupStepOutcome, diff --git a/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts b/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts index e4a0fb0b3719..e159338806b9 100644 --- a/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts +++ b/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts @@ -6,7 +6,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type * as AgentStatusModule from '@/lib/agent-status' import { buildWorkspaceSessionPayload } from '@/lib/workspace-session' import type { Tab } from '../../../../shared/tab-types' -import { orderOnScreenStructuredAgentSessionsFirst } from '../../../../main/runtime/saved-structured-agent-session-restoration' +import { orderOnScreenStructuredAgentSessionsFirst } from '../../../../shared/saved-on-screen-structured-agent-sessions' import { createTabsSliceMockApi } from './tabs-slice-test-harness' import { createTestStore } from './store-test-helpers' import { buildActiveSurfacePatch } from './tabs/tabs-surface' diff --git a/src/shared/saved-on-screen-structured-agent-sessions.ts b/src/shared/saved-on-screen-structured-agent-sessions.ts new file mode 100644 index 000000000000..4a24d7d465ec --- /dev/null +++ b/src/shared/saved-on-screen-structured-agent-sessions.ts @@ -0,0 +1,44 @@ +// Which structured chats a launch shows first, from the workspace session the renderer saves. + +import { LOCAL_EXECUTION_HOST_ID } from './execution-host' +import type { WorkspaceSessionState } from './workspace-session-state-types' + +/** + * The chats a launch shows first, each tab group's active tab (the active worktree's first, and in + * each worktree its focused group first), ahead of the rest in their given order. Read from the + * groups because `activeTabIdByWorktree` only ever holds a terminal's tab. Any agent: this orders, + * it restores nothing. + */ +export function orderOnScreenStructuredAgentSessionsFirst( + sessionIds: readonly string[], + session: WorkspaceSessionState | null +): string[] { + const tabsById = new Map( + Object.values(session?.unifiedTabs ?? {}) + .flat() + .map((tab) => [tab.id, tab]) + ) + const groupsByWorktree = session?.tabGroups ?? {} + const activeWorktree = session?.activeWorktreeId ?? null + const worktrees = [ + ...(activeWorktree && groupsByWorktree[activeWorktree] ? [activeWorktree] : []), + ...Object.keys(groupsByWorktree).filter((worktreeId) => worktreeId !== activeWorktree) + ] + const listed = new Set(sessionIds) + const first = new Set() + for (const worktreeId of worktrees) { + const groups = groupsByWorktree[worktreeId] ?? [] + const focused = session?.activeGroupIdByWorktree?.[worktreeId] + for (const group of [ + ...groups.filter((candidate) => candidate.id === focused), + ...groups.filter((candidate) => candidate.id !== focused) + ]) { + const tab = group.activeTabId ? tabsById.get(group.activeTabId) : undefined + const local = !tab?.executionHostId || tab.executionHostId === LOCAL_EXECUTION_HOST_ID + if (tab?.contentType === 'agent-session' && local && listed.has(tab.entityId)) { + first.add(tab.entityId) + } + } + } + return [...first, ...sessionIds.filter((sessionId) => !first.has(sessionId))] +} From 98eeaa2591298543217afb3054f6ecb1d176e5ac Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:55:13 -0700 Subject: [PATCH 069/107] fix(native-chat): a corrupt rowless chat gets no row from the startup pass, so its rebuild still runs Only an unfinished chat's row is stored unshown. A corrupt history's settled-looking row would be seeded on the next boot and take the chat out of the background open its rebuild needs, so it gets no row, as before the slice batching, and every boot opens it until its open rebuilds it. --- .../structured-agent-session-startup-state.ts | 5 +- ...red-agent-session-startup-status-derive.ts | 22 ++++--- ...ctured-agent-session-status-derive.test.ts | 61 +++++++++++++++++++ 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index d7ba67bad4ef..bdb8846fdea0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -82,8 +82,9 @@ export type StructuredAgentSessionStartupState = { * The background pass's first step: each listed chat with history here and no status row gets * its row from its rows alone, opening nothing, folded one chat at a time in the order given (the * startup step puts the chats on screen first) and written a slice at a time. A row showing work - * a gone process left, or a corrupt history, is stored but never shown. Answers the chats still - * to open, which their open settles, rebuilds or gives a row; see `deriveMissingStatuses`. + * a gone process left is stored but never shown; a corrupt history gets no row, so its open and + * rebuild run. Answers the chats still to open, which their open settles, rebuilds or gives a + * row; see `deriveMissingStatuses`. */ deriveMissingStatuses: (sessionIds: readonly string[]) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts index f6f0503c0ff4..6729046fdabf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts @@ -16,12 +16,14 @@ import type { StructuredAgentSessionStartupStateDeps } from './structured-agent- const SLICE_CHATS = 16 const SLICE_MS = 50 -/** `shown` false: a row stored so the next boot selects the chat, but never displayed from rows. */ +/** `shown` false: an unfinished chat's row, stored so the next boot selects the chat until its open + * settles it, but never displayed from rows. */ type Folded = FoldedJournalSessionStatus & { record: AgentSessionRecord; shown: boolean } -/** Answers the chats still to open: the chats it could not fold, and those whose rows show work a - * gone process left or owe a rebuild, which their open settles or rebuilds, in the order given; - * then, after each slice's write, the chats of that slice the write had to skip. */ +/** Answers the chats still to open: the chats it could not fold, those with a corrupt history + * (given no row, so every boot opens one until its rebuild), and those whose rows show work a gone + * process left, which their open settles, in the order given; then, after each slice's write, the + * chats of that slice the write had to skip. */ export async function deriveMissingStatuses( deps: StructuredAgentSessionStartupStateDeps, sessionIds: readonly string[] @@ -73,9 +75,10 @@ export async function deriveMissingStatuses( return toOpen } -/** A chat's status from its rows; null leaves it to its open. One with work a gone process left, - * or a corrupt history, is still written so startup selects it until its open settles it, but it - * is never shown: only its open, which settles or rebuilds it, publishes its status. */ +/** A chat's status from its rows; null leaves it to its open, rowless, as for a corrupt history: a + * stored row would let the next boot seed it and skip the open its rebuild needs. One with work a + * gone process left is still written, so startup selects it until its open settles it, but it is + * never shown: only its open publishes its status. */ async function foldFromRows( deps: StructuredAgentSessionStartupStateDeps, record: AgentSessionRecord, @@ -87,11 +90,10 @@ async function foldFromRows( record.sessionId, options ) - if (!folded) { + if (!folded || folded.load.corrupt) { return null } - const shown = !folded.load.corrupt && !isUnsettledJournalSessionStatus(folded.status) - return { ...folded, record, shown } + return { ...folded, record, shown: !isUnsettledJournalSessionStatus(folded.status) } } catch (error) { deps.openDeps.logger.warn('deriving a chat status from its rows failed', { scope: 'startup-status-derive', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts index b03ecce6e5e2..796a913511ff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts @@ -8,10 +8,13 @@ import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION } from '../../../shared/agent-session-journal-types' import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' +import type * as JournalRecoveryModule from './agent-session-journal-recovery' import { closeTestJournalHostDatabases, insertTestJournalRow, + insertTestJournalRowJson, liveTestJournalRows, + loadTestJournal, openTestJournalHostDatabase, readTestJournalSessionStatus } from '../agent-session-journal/journal-host-database-test-support' @@ -22,6 +25,7 @@ import { type RestTestRig } from './structured-agent-session-rest-test-rig' import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' +import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { latestRestTestStatus, restTestOpens @@ -52,11 +56,34 @@ vi.mock('../agent-session-journal/journal-session-status-backfill', async (impor } }) +// Each chat an open found corrupt and sent through its rebuild. +const rebuilds = vi.hoisted(() => { + const opened: { sessionIds: string[] } = { sessionIds: [] } + return opened +}) + +vi.mock('./agent-session-journal-recovery', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + openAgentSessionJournalWithRecovery: async ( + ...args: Parameters + ) => { + const opened = await actual.openAgentSessionJournalWithRecovery(...args) + if (opened.recovery?.trigger === 'journal_corrupt') { + rebuilds.sessionIds.push(args[0].identity.sessionId) + } + return opened + } + } +}) + const rigs: RestTestRig[] = [] afterEach(async () => { folds.done.clear() folds.after = null + rebuilds.sessionIds.length = 0 for (const rig of rigs.splice(0)) { await rig.dispose() } @@ -405,4 +432,38 @@ describe('rowless listed chats after an upgrade', () => { lifecycle: 'idle' }) }) + + it('gives a corrupt rowless chat no row, so the next boot still opens it and its rebuild runs', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-corrupt', { message: 'asked' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + const { db } = openTestJournalHostDatabase(rig.root) + const tip = liveTestJournalRows(db, 'session-corrupt').at(-1)! + insertTestJournalRowJson(db, 'session-corrupt', tip.seq + 1, '{') + upgradeToEmptyStatusTable(rig) + expect(loadTestJournal(rig.root, 'session-corrupt')).toMatchObject({ corrupt: true }) + await rig.boot() + // Orca quits after the pass's rows, before the restore that follows opens anything. + const restore = vi + .spyOn(StructuredAgentSessionReadableRestorer.prototype, 'restore') + .mockResolvedValue(undefined) + + await rig.host.restoreReadableSessions(await startupThroughListing(rig)) + + expect(folds.done.has('session-corrupt')).toBe(true) + expect(restore).toHaveBeenCalledWith(['session-corrupt']) + expect(readTestJournalSessionStatus(rig.root, 'session-corrupt')).toBeNull() + expect(seeded(rig, 'session-corrupt')).toBe(-1) + + restore.mockRestore() + await rig.crash() + await rig.boot() + const background = await startupThroughListing(rig) + expect(background).toEqual(['session-corrupt']) + await rig.host.restoreReadableSessions(background) + + expect(restTestOpens(rig, 'session-corrupt')).toBeGreaterThan(0) + expect(rebuilds.sessionIds).toEqual(['session-corrupt']) + }) }) From 39ef3764b0be468a98ff274307c60973ac9aa560 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:19:15 -0700 Subject: [PATCH 070/107] refactor(native-chat): drop the single-chat status backfill; callers fold and write a batch themselves --- .../journal-session-status-backfill.test.ts | 43 ++++++++++++++----- .../journal-session-status-backfill.ts | 18 +------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index 1428c7c3af1e..c979f1d2400e 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -6,7 +6,9 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' import { + closeTestJournalHostDatabase, createTrackedJournalOpener, insertTestJournalRowJson, publishTestJournalEpoch, @@ -15,7 +17,6 @@ import { } from './journal-host-database-test-support' import * as JournalOpen from './journal-open' import { - backfillJournalSessionStatus, foldJournalSessionStatus, writeJournalSessionStatuses, type FoldedJournalSessionStatus @@ -60,6 +61,15 @@ const database = () => openTestJournalHostDatabase(root) const dropRow = (sessionId: string) => database().db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) +/** The chat's fold, and its row written as a batch of one; null when either declines. */ +async function foldAndWrite( + sessionId: string, + options?: Parameters[2] +): Promise { + const folded = await foldJournalSessionStatus(database(), sessionId, options) + return folded && writeJournalSessionStatuses(database(), [folded]).length > 0 ? folded : null +} + beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-status-backfill-')) clock = 1_000 @@ -84,13 +94,13 @@ describe('a row from the rows alone', () => { await journals.closeAll() dropRow(name) - const derived = await backfillJournalSessionStatus(database(), name) + const derived = await foldAndWrite(name) expect(derived?.status ?? null).toEqual(byOpen) expect(readTestJournalSessionStatus(root, name)).toEqual(byOpen) }) - it('equals the row an open writes for a chat whose history is corrupt', async () => { + it('folds a chat whose history is corrupt to the status an open writes, and writes nothing', async () => { const journal = await open('corrupt') await JOURNAL_SESSION_STATE_CORPUS['working subagent roster'](journal) const tip = journal.cursor() @@ -103,12 +113,25 @@ describe('a row from the rows alone', () => { await journals.closeAll() dropRow('corrupt') - const derived = await backfillJournalSessionStatus(database(), 'corrupt') + // Folded only: the startup pass leaves a corrupt chat rowless, for its open to rebuild. + const derived = await foldJournalSessionStatus(database(), 'corrupt') // The rebuild the corruption owes decides the roster, so neither row counts it as live work. expect(derived?.load.corrupt).toBe(true) expect(byOpen?.liveChildWork).toBe(false) expect(derived?.status ?? null).toEqual(byOpen) + expect(readTestJournalSessionStatus(root, 'corrupt')).toBeNull() + }) + + it("folds nothing from a newer build's database", async () => { + await JOURNAL_SESSION_STATE_CORPUS.settled(await open('newer')) + await journals.closeAll() + dropRow('newer') + database().db.pragma(`user_version = ${JOURNAL_DB_SCHEMA_VERSION + 1}`) + closeTestJournalHostDatabase(root) + expect(database().readOnly).toBe(true) + + expect(await foldJournalSessionStatus(database(), 'newer')).toBeNull() }) it('folds a large chat a bounded part per task', async () => { @@ -139,7 +162,7 @@ describe('a row from the rows alone', () => { }) const budget = 20_000 - const derived = await backfillJournalSessionStatus(database(), 'large', { + const derived = await foldAndWrite('large', { batchRows: 64, batchChars: budget, yieldTask: async () => { @@ -161,7 +184,7 @@ describe('a row from the rows alone', () => { dropRow('moving') // An append lands between parts: the fold read a tip the chat has left. let appended = false - const moved = await backfillJournalSessionStatus(database(), 'moving', { + const moved = await foldAndWrite('moving', { batchRows: 1, yieldTask: async () => { if (!appended) { @@ -180,7 +203,7 @@ describe('a row from the rows alone', () => { expect(readTestJournalSessionStatus(root, 'moving')).toBeNull() const quit = new AbortController() - const stopped = await backfillJournalSessionStatus(database(), 'moving', { + const stopped = await foldJournalSessionStatus(database(), 'moving', { batchRows: 1, signal: quit.signal, yieldTask: async () => quit.abort() @@ -189,13 +212,13 @@ describe('a row from the rows alone', () => { expect(readTestJournalSessionStatus(root, 'moving')).toBeNull() // Still in a per-chat file, or never written: no epoch here. - expect(await backfillJournalSessionStatus(database(), 'never-written')).toBeNull() + expect(await foldJournalSessionStatus(database(), 'never-written')).toBeNull() - await backfillJournalSessionStatus(database(), 'moving') + await foldAndWrite('moving') const row = readTestJournalSessionStatus(root, 'moving') expect(row).not.toBeNull() // A row already there is never rewritten. - expect(await backfillJournalSessionStatus(database(), 'moving')).toBeNull() + expect(await foldJournalSessionStatus(database(), 'moving')).toBeNull() }) it('writes a whole batch in one transaction, each row the row an open writes', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index a0b91486932e..03bf61bc17fe 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -3,8 +3,8 @@ // Version 5 creates the status table empty, so after an upgrade every chat has no row until // something writes it. Rather than open each chat (its lease, settle and conversation), its rows are // folded a bounded part per task, as a replay folds them, then the row is derived and written in one -// short transaction, only while the chat is still where the fold read it. The startup pass folds a -// slice of chats and writes their rows in one transaction; the background copy writes one at a time. +// short transaction, only while the chat is still where the fold read it. Callers fold chats one at a +// time and write a slice of their rows in one transaction. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { JournalHostDatabase } from './journal-host-database' @@ -63,20 +63,6 @@ type FoldOptions = { signal?: AbortSignal } -/** The chat's row and status: folded, then written alone. Null when something wrote the row or - * moved the chat first, or as `foldJournalSessionStatus` says. */ -export async function backfillJournalSessionStatus( - database: JournalHostDatabase, - sessionId: string, - options: FoldOptions = {} -): Promise<{ load: JournalLoad; status: JournalSessionStatus } | null> { - const folded = await foldJournalSessionStatus(database, sessionId, options) - if (!folded || options.signal?.aborted) { - return null - } - return writeJournalSessionStatuses(database, [folded]).length > 0 ? folded : null -} - /** The chat's rows folded a part per task, as a replay folds them, and its status derived. Null when * the chat already has a row, has no epoch in this database (it is still in a per-chat file), the * database or the chat's rows are a newer build's, or `signal` aborted, which is checked before each From 226da57875eadec5a4e8059c60a639a219211099 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:15:19 -0700 Subject: [PATCH 071/107] perf(native-chat): the startup pass commits its status-row slices unsynced --- .../journal-session-status-backfill.test.ts | 38 +++++++++++++++++++ .../journal-session-status-backfill.ts | 3 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index c979f1d2400e..20cbdbea6768 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -249,6 +249,44 @@ describe('a row from the rows alone', () => { } }) + it('commits a batch unsynced and restores the synced level after, also when the commit fails', async () => { + for (const sessionId of ['first', 'second']) { + await JOURNAL_SESSION_STATE_CORPUS.settled(await open(sessionId)) + } + await journals.closeAll() + const folded: FoldedJournalSessionStatus[] = [] + for (const sessionId of ['first', 'second']) { + dropRow(sessionId) + folded.push((await foldJournalSessionStatus(database(), sessionId))!) + } + const connection = database().db + const level = () => Number(connection.pragma('synchronous', { simple: true })) + const exec = connection.exec.bind(connection) + const atCommit: number[] = [] + let failCommit = false + vi.spyOn(connection, 'exec').mockImplementation((sql) => { + if (sql === 'COMMIT') { + atCommit.push(level()) + if (failCommit) { + throw new Error('disk I/O error') + } + } + exec(sql) + }) + // NORMAL (1) for the batch's commit, FULL (2) for every other commit on the connection. + expect(level()).toBe(2) + + expect(writeJournalSessionStatuses(database(), [folded[0]!])).toHaveLength(1) + expect(atCommit).toEqual([1]) + expect(level()).toBe(2) + + failCommit = true + expect(() => writeJournalSessionStatuses(database(), [folded[1]!])).toThrow('disk I/O error') + expect(connection.isTransaction).toBe(false) + expect(level()).toBe(2) + expect(readTestJournalSessionStatus(root, 'second')).toBeNull() + }) + it('skips only the chats in a batch that moved, got a row or a new history since their fold', async () => { const ids = ['steady', 'appended', 'rowed', 'replaced'] const journalsById = new Map() diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index 03bf61bc17fe..7719b18a99ab 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -135,7 +135,8 @@ export function writeJournalSessionStatuses( if (folded.length === 0) { return [] } - return database.transaction((db) => + // Unsynced: a lost row is re-derived from the journal, and any later synced commit covers it. + return database.unsyncedTransaction((db) => folded.filter(({ sessionId, epoch, tip, status }) => { if ( hasJournalSessionStatus(db, sessionId) || From f62792bf603ec11414abdb71f60c9b7cdced33d5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:55:27 -0700 Subject: [PATCH 072/107] test(native-chat): after an unsynced status slice the next commit is synced, and a lost row folds again to the same row --- .../journal-session-status-backfill.test.ts | 43 ++++++++++++++++ ...ctured-agent-session-status-derive.test.ts | 51 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index 20cbdbea6768..eea360a4c2be 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -287,6 +287,49 @@ describe('a row from the rows alone', () => { expect(readTestJournalSessionStatus(root, 'second')).toBeNull() }) + it("leaves the next commit synced, a chat's own append included, after a slice that landed or failed", async () => { + for (const sessionId of ['first', 'second']) { + await JOURNAL_SESSION_STATE_CORPUS.settled(await open(sessionId)) + } + const live = await open('live') + await JOURNAL_SESSION_STATE_CORPUS.settled(live) + const folded: FoldedJournalSessionStatus[] = [] + for (const sessionId of ['first', 'second']) { + dropRow(sessionId) + folded.push((await foldJournalSessionStatus(database(), sessionId))!) + } + const connection = database().db + const exec = connection.exec.bind(connection) + const atCommit: number[] = [] + let failNextCommit = false + vi.spyOn(connection, 'exec').mockImplementation((sql) => { + if (sql === 'COMMIT') { + atCommit.push(Number(connection.pragma('synchronous', { simple: true }))) + if (failNextCommit) { + failNextCommit = false + throw new Error('disk I/O error') + } + } + exec(sql) + }) + const append = (text: string) => + live.appendItem( + { provider: 'orca', clientMessageId: text }, + { kind: 'status', text }, + { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } + ) + + writeJournalSessionStatuses(database(), [folded[0]!]) + await append('after a slice') + failNextCommit = true + expect(() => writeJournalSessionStatuses(database(), [folded[1]!])).toThrow('disk I/O error') + await append('after a failed slice') + database().transaction((db) => db.prepare('SELECT 1').get()) + + // The slices at NORMAL (1); every commit after them, the chat's appends first, at FULL (2). + expect(atCommit).toEqual([1, 2, 1, 2, 2]) + }) + it('skips only the chats in a batch that moved, got a row or a new history since their fold', async () => { const ids = ['steady', 'appended', 'rowed', 'replaced'] const journalsById = new Map() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts index 796a913511ff..dd815b248938 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts @@ -466,4 +466,55 @@ describe('rowless listed chats after an upgrade', () => { expect(restTestOpens(rig, 'session-corrupt')).toBeGreaterThan(0) expect(rebuilds.sessionIds).toEqual(['session-corrupt']) }) + + it('folds a chat again after its unsynced row is lost, to the same row and no wrong status', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-settled', { message: 'asked session-settled' }) + await crashMidTurn(rig, 'session-unfinished') + const ids = ['session-settled', 'session-unfinished'] + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + // Only the pass's rows: no open, so the unfinished chat keeps the row the pass stored. + const restore = vi + .spyOn(StructuredAgentSessionReadableRestorer.prototype, 'restore') + .mockResolvedValue(undefined) + await rig.host.restoreReadableSessions(await startupThroughListing(rig)) + const first = ids.map((sessionId) => readTestJournalSessionStatus(rig.root, sessionId)) + const firstStatus = latestRestTestStatus(rig, 'session-settled') + expect(first[0]).toMatchObject({ lifecycle: 'idle' }) + expect(first[1]).toMatchObject({ lifecycle: 'running' }) + + // The unsynced commit is lost: a power cut, then a boot that finds neither row. + await rig.crash() + upgradeToEmptyStatusTable(rig) + folds.done.clear() + await rig.boot() + const eventsBefore = rig.statusEvents.length + await rig.host.restoreReadableSessions(await startupThroughListing(rig)) + + expect(ids.every((sessionId) => folds.done.has(sessionId))).toBe(true) + expect(ids.map((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toEqual(first) + // Nothing published in between but the settled chat's own row, as the first boot published it. + const between = rig.statusEvents.slice(eventsBefore) + expect( + between.every( + (event) => event.type !== 'status' || event.session.sessionId === 'session-settled' + ) + ).toBe(true) + expect(latestRestTestStatus(rig, 'session-settled')).toEqual(firstStatus) + + // Each row equals what an open writes for the chat. + restore.mockRestore() + upgradeToEmptyStatusTable(rig) + for (const sessionId of ids) { + await rig.host.history({ sessionId, direction: 'tail' }) + } + expect(readTestJournalSessionStatus(rig.root, 'session-settled')).toEqual(first[0]) + // The open settles what the pass only stored: never shown as running. + expect(readTestJournalSessionStatus(rig.root, 'session-unfinished')).toMatchObject({ + lifecycle: 'idle' + }) + expect(latestRestTestStatus(rig, 'session-unfinished')?.status).not.toBe('working') + }) }) From 161001fdef6ef6a5e1726fef3fb37b73b9a0823d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:23:51 -0700 Subject: [PATCH 073/107] perf(native-chat): the startup status pass yields only before chats it folds, and once per 8 ms run of the rest --- .../journal-session-status-backfill.ts | 20 +- ...gent-session-startup-status-derive.test.ts | 185 ++++++++++++++++++ ...red-agent-session-startup-status-derive.ts | 44 ++++- 3 files changed, 238 insertions(+), 11 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index 7719b18a99ab..a78d8992c3b1 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -63,6 +63,19 @@ type FoldOptions = { signal?: AbortSignal } +/** The epoch a fold of the chat would read, or null when it has nothing to fold: the database is a + * newer build's, the chat has no epoch here (it is still in a per-chat file), or it has a row. */ +export function foldableJournalSessionEpoch( + database: JournalHostDatabase, + sessionId: string +): string | null { + if (database.readOnly) { + return null + } + const epoch = readJournalSessionEpoch(database.db, sessionId) + return epoch === null || hasJournalSessionStatus(database.db, sessionId) ? null : epoch +} + /** The chat's rows folded a part per task, as a replay folds them, and its status derived. Null when * the chat already has a row, has no epoch in this database (it is still in a per-chat file), the * database or the chat's rows are a newer build's, or `signal` aborted, which is checked before each @@ -77,11 +90,8 @@ export async function foldJournalSessionStatus( signal }: FoldOptions = {} ): Promise { - if (database.readOnly) { - return null - } - const epoch = readJournalSessionEpoch(database.db, sessionId) - if (epoch === null || hasJournalSessionStatus(database.db, sessionId)) { + const epoch = foldableJournalSessionEpoch(database, sessionId) + if (epoch === null) { return null } const tip = readJournalTip(database.db, sessionId, epoch) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts new file mode 100644 index 000000000000..9a367c895499 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts @@ -0,0 +1,185 @@ +// The startup status pass's tasks: a chat with nothing to fold (no history in the host's database +// yet, or a row already) goes by without a task of its own, a long run of them still yields, and a +// chat that folds always starts a task. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type * as TimersPromises from 'node:timers/promises' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' +import { + closeTestJournalHostDatabases, + createTrackedJournalOpener, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from '../agent-session-journal/journal-host-database-test-support' +import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' +import { + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CORPUS +} from '../agent-session-journal/journal-session-state-test-corpus' +import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' + +// What the pass did, in order: each yield to the event loop, and each fold it started. +const trace = vi.hoisted(() => { + const log: { events: string[] } = { events: [] } + return log +}) + +vi.mock('node:timers/promises', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + setImmediate: (...args: Parameters) => { + trace.events.push('yield') + return actual.setImmediate(...args) + } + } +}) + +vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + foldJournalSessionStatus: (...args: Parameters) => { + trace.events.push(`fold:${args[1]}`) + return actual.foldJournalSessionStatus(...args) + } + } +}) + +import { deriveMissingStatuses } from './structured-agent-session-startup-status-derive' + +const journals = createTrackedJournalOpener() +let root: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-status-derive-')) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +function record(sessionId: string): AgentSessionRecord { + return { ...agentSessionRecordFixture(), sessionId } +} + +/** Chats with a settled history in the host's database and no status row: each one folds. */ +async function chatsWithHistory(sessionIds: readonly string[]): Promise { + for (const sessionId of sessionIds) { + const journal = await journals.open({ + identity: { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + }, + stateDirectory: root, + mintEpoch: () => `epoch-${sessionId}`, + currentFence: () => CORPUS_FENCE + }) + await JOURNAL_SESSION_STATE_CORPUS.settled(journal) + } + await journals.closeAll() + openTestJournalHostDatabase(root).db.exec('DELETE FROM journal_session_state') +} + +function deps(seeded: string[] = []): StructuredAgentSessionStartupStateDeps { + const stand = { + openDeps: { + journalDatabase: openTestJournalHostDatabase(root), + store: { getRecord: record, listRecords: () => [] }, + logger: { warn: vi.fn(), error: vi.fn() } + }, + canSettle: (candidate: AgentSessionRecord | null): candidate is AgentSessionRecord => + candidate !== null, + seedStatus: (chat: AgentSessionRecord) => { + seeded.push(chat.sessionId) + }, + resolveRecovery: async () => true, + restoreListed: async () => undefined, + serialize: (_sessionId: string, task: () => Promise) => task(), + hasSession: () => false, + isListed: () => true, + isDisposed: () => false + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pass reads only the database, records, logger and the members above. + return stand as unknown as StructuredAgentSessionStartupStateDeps +} + +const yields = () => trace.events.filter((event) => event === 'yield').length + +describe('the startup status pass over chats with nothing to fold', () => { + it('passes them all, in order, without a task each and without a write', async () => { + const ids = Array.from({ length: 100 }, (_, index) => `session-file-${index}`) + const stand = deps() + const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') + trace.events = [] + + const toOpen = await deriveMissingStatuses(stand, ids) + + expect(toOpen).toEqual(ids) + expect(trace.events.filter((event) => event.startsWith('fold:'))).toEqual([]) + expect(yields()).toBeLessThan(ids.length / 4) + // An empty slice commits nothing. + expect(transaction).not.toHaveBeenCalled() + }) + + it('still yields during a long run of them', async () => { + const ids = Array.from({ length: 60 }, (_, index) => `session-file-${index}`) + const stand = deps() + // Each clock read moves 3 ms: a run of skips crosses the 8 ms bound every few chats. + let clock = 0 + vi.spyOn(performance, 'now').mockImplementation(() => (clock += 3)) + trace.events = [] + + expect(await deriveMissingStatuses(stand, ids)).toEqual(ids) + + expect(yields()).toBeGreaterThanOrEqual(10) + expect(yields()).toBeLessThan(ids.length) + }) + + it('keeps the order in a mixed pass and starts a task before each chat that folds', async () => { + await chatsWithHistory(['session-fold-1', 'session-fold-2']) + const ids = [ + 'session-file-1', + 'session-fold-1', + 'session-file-2', + 'session-file-3', + 'session-fold-2', + 'session-file-4' + ] + const seeded: string[] = [] + const stand = deps(seeded) + trace.events = [] + + const toOpen = await deriveMissingStatuses(stand, ids) + + expect(toOpen).toEqual(['session-file-1', 'session-file-2', 'session-file-3', 'session-file-4']) + expect(seeded).toEqual(['session-fold-1', 'session-fold-2']) + for (const sessionId of ['session-fold-1', 'session-fold-2']) { + expect(readTestJournalSessionStatus(root, sessionId)).toMatchObject({ lifecycle: 'idle' }) + } + // A yield between the start (or the previous fold) and each fold. + let sinceFold: string[] = [] + for (const event of trace.events) { + if (event.startsWith('fold:')) { + expect(sinceFold).toContain('yield') + sinceFold = [] + } else { + sinceFold.push(event) + } + } + expect(trace.events.filter((event) => event.startsWith('fold:'))).toEqual([ + 'fold:session-fold-1', + 'fold:session-fold-2' + ]) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts index 6729046fdabf..b295b03d1310 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts @@ -5,6 +5,7 @@ import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { + foldableJournalSessionEpoch, foldJournalSessionStatus, writeJournalSessionStatuses, type FoldedJournalSessionStatus @@ -15,6 +16,8 @@ import type { StructuredAgentSessionStartupStateDeps } from './structured-agent- // A write per slice, not per chat: each per-chat commit rewrites the same table and index pages. const SLICE_CHATS = 16 const SLICE_MS = 50 +// Chats with nothing to fold go by without a yield each; a run of them still yields once this long. +const SKIP_RUN_MS = 8 /** `shown` false: an unfinished chat's row, stored so the next boot selects the chat until its open * settles it, but never displayed from rows. */ @@ -42,17 +45,30 @@ export async function deriveMissingStatuses( const toOpen: string[] = [] let slice: Folded[] = [] let sliceStart = 0 + let lastYield = performance.now() for (const sessionId of sessionIds) { - // A task per chat at least: short chats fold in one part, and a pass of them must not be one task. + const record = deps.openDeps.store.getRecord(sessionId) + if (!record || !canFold(deps, record)) { + if (performance.now() - lastYield >= SKIP_RUN_MS) { + await yieldTask() + lastYield = performance.now() + } + if (deps.isDisposed()) { + return [] + } + toOpen.push(sessionId) + continue + } + // A task per chat that folds: short chats fold in one part, and a pass of them must not be one task. await yieldTask() + lastYield = performance.now() if (deps.isDisposed()) { return [] } - const record = deps.openDeps.store.getRecord(sessionId) - const folded = - record && deps.canSettle(record) && !deps.hasSession(sessionId) - ? await foldFromRows(deps, record, { yieldTask, signal: quit.signal }) - : null + // Opened during the yield: its open writes and publishes its row. + const folded = deps.hasSession(sessionId) + ? null + : await foldFromRows(deps, record, { yieldTask, signal: quit.signal }) if (!folded || !folded.shown) { toOpen.push(sessionId) } @@ -75,6 +91,22 @@ export async function deriveMissingStatuses( return toOpen } +/** Whether the chat has rows to fold here, decided without a task of its own. A check that throws + * answers yes, so the fold reports the failure. */ +function canFold( + deps: StructuredAgentSessionStartupStateDeps, + record: AgentSessionRecord +): boolean { + if (!deps.canSettle(record) || deps.hasSession(record.sessionId)) { + return false + } + try { + return foldableJournalSessionEpoch(deps.openDeps.journalDatabase, record.sessionId) !== null + } catch { + return true + } +} + /** A chat's status from its rows; null leaves it to its open, rowless, as for a corrupt history: a * stored row would let the next boot seed it and skip the open its rebuild needs. One with work a * gone process left is still written, so startup selects it until its open settles it, but it is From 414bf53c87abbcbd34dd64a9eeaabaaaf977972a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:58:28 -0700 Subject: [PATCH 074/107] test(native-chat): a paired client whose first tab listing times out still mirrors the host's chats from the stream --- ...on-tabs-sync-slow-initial-listing.test.tsx | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx diff --git a/src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx b/src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx new file mode 100644 index 000000000000..fa1e6e9f6482 --- /dev/null +++ b/src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx @@ -0,0 +1,200 @@ +// @vitest-environment happy-dom + +// A paired host's first tab listing can take longer than the client waits for it: on the first +// boot after an upgrade the host derives its chats' statuses before it answers. The listing's +// timeout must not leave the client without the host's tabs. The all-worktrees stream, open since +// the same moment, delivers the host's full inventory as its first frame once the host is ready. + +import { act, cleanup, renderHook } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RuntimeMobileSessionTabsResult } from '../../../shared/runtime-types' +import type { RuntimeRpcResponse } from '../../../shared/runtime-rpc-envelope' +import type * as WorktreeRuntimeOwnerModule from '@/lib/worktree-runtime-owner' + +const mocks = vi.hoisted(() => ({ + getExplicitRuntimeEnvironmentIdForWorktree: vi.fn(), + recoverSnapshot: vi.fn(), + runtimeSessionMirrorEnvironmentKey: vi.fn() +})) + +vi.mock('./use-runtime-session-mirror-environment-key', () => ({ + useRuntimeSessionMirrorEnvironmentKeys: () => ({ + environmentKey: mocks.runtimeSessionMirrorEnvironmentKey(), + resubscribeSignal: '' + }) +})) + +vi.mock('@/lib/worktree-runtime-owner', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + getExplicitRuntimeEnvironmentIdForWorktree: mocks.getExplicitRuntimeEnvironmentIdForWorktree + } +}) + +vi.mock('./web-session-terminal-orphan-recovery', () => ({ + recoverWebSessionTerminalOrphansBeforeApply: mocks.recoverSnapshot +})) + +import { useAppStore } from '@/store' +import type { PublicKnownRuntimeEnvironment } from '../../../shared/runtime-environments' +import type { AppState } from '@/store/types' +import { replaceRuntimeEnvironmentRevisions } from './runtime-environment-revision' +import { + resetWebSessionTabsSnapshotFreshnessForTests, + useWebSessionTabsSync +} from './web-session-tabs-sync' + +const ENV_A = 'env-a' +const WORKTREE = 'repo-a::worktree-a' +const REVISION_A = 101 +const MIRROR_KEY = `${ENV_A}\u0001runtime-a\u00011\u0001${REVISION_A}` +const initialState = useAppStore.getInitialState() + +type RuntimeCall = typeof window.api.runtimeEnvironments.call +type RuntimeSubscribe = typeof window.api.runtimeEnvironments.subscribe +type RuntimeSubscription = { + request: Parameters[0] + callbacks: Parameters[1] +} + +const subscriptions: RuntimeSubscription[] = [] +// The listing in flight, answered by the test: here, only ever by the client's own timeout. +let answerListing: (response: RuntimeRpcResponse) => void = () => {} +const runtimeCall = vi.fn( + (request) => + new Promise((resolve) => { + if (request.method === 'session.tabs.listAll') { + answerListing = resolve + } + }) +) +const runtimeSubscribe = vi.fn(async (request, callbacks) => { + subscriptions.push({ request, callbacks }) + return { unsubscribe: vi.fn(), sendBinary: vi.fn() } +}) + +function chatSnapshot(): RuntimeMobileSessionTabsResult { + return { + worktree: WORKTREE, + publicationEpoch: 'structured:epoch-1', + snapshotVersion: 1, + activeGroupId: 'host-group-1', + activeTabId: 'agent-session:codex-1', + activeTabType: 'agent-session', + tabs: [ + { + type: 'agent-session', + id: 'agent-session:codex-1', + title: 'Codex Chat', + sessionId: 'codex-1', + agent: 'codex', + isActive: true + } + ] + } +} + +async function settle(): Promise { + for (let index = 0; index < 6; index += 1) { + await Promise.resolve() + } +} + +function inventoryStream(): RuntimeSubscription { + const stream = subscriptions.find( + ({ request }) => request.method === 'session.tabs.subscribeAll' && request.selector === ENV_A + ) + if (!stream) { + throw new Error('the all-worktrees stream was never opened') + } + return stream +} + +const mirroredChats = () => + (useAppStore.getState().unifiedTabsByWorktree[WORKTREE] ?? []).filter( + (tab) => tab.contentType === 'agent-session' + ) + +describe('a paired host whose first tab listing outlasts the client', () => { + beforeEach(() => { + subscriptions.length = 0 + runtimeCall.mockClear() + runtimeSubscribe.mockClear() + mocks.recoverSnapshot.mockReset().mockImplementation(async (_state, snapshot) => snapshot) + mocks.getExplicitRuntimeEnvironmentIdForWorktree.mockReset().mockReturnValue(ENV_A) + mocks.runtimeSessionMirrorEnvironmentKey.mockReset().mockReturnValue(MIRROR_KEY) + Object.defineProperty(window, 'api', { + configurable: true, + value: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } } + }) + resetWebSessionTabsSnapshotFreshnessForTests() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mirror scan and revision ledger read only id, createdAt and pairingRevision. + const runtimeEnvironments = [ + { id: ENV_A, createdAt: 100, pairingRevision: REVISION_A } + ] as PublicKnownRuntimeEnvironment[] + replaceRuntimeEnvironmentRevisions(runtimeEnvironments) + useAppStore.setState( + { + ...initialState, + activeWorktreeId: WORKTREE, + workspaceSessionReady: true, + runtimeEnvironments, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mirror reads only runtimeId and connectionGeneration from each status. + runtimeStatusByEnvironmentId: new Map([ + [ENV_A, { status: { runtimeId: 'runtime-a' }, connectionGeneration: 1 }] + ]) as AppState['runtimeStatusByEnvironmentId'] + }, + true + ) + }) + + afterEach(() => { + cleanup() + useAppStore.setState(initialState, true) + replaceRuntimeEnvironmentRevisions([]) + resetWebSessionTabsSnapshotFreshnessForTests() + }) + + it("still mirrors the host's chats from the stream after the listing times out", async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + renderHook(() => useWebSessionTabsSync()) + await act(settle) + expect(runtimeCall).toHaveBeenCalledWith( + expect.objectContaining({ method: 'session.tabs.listAll', timeoutMs: 15_000 }) + ) + const stream = inventoryStream() + + // 15 s pass with the host still deriving: the client gives up on its listing. + await act(async () => { + answerListing({ + id: 'list-all', + ok: false, + error: { + code: 'runtime_timeout', + message: 'Timed out waiting for the remote Orca runtime response.' + }, + _meta: { runtimeId: 'runtime-a' } + }) + await settle() + }) + expect(mirroredChats()).toEqual([]) + + // The host finishes: the stream's first frame is its whole inventory. + await act(async () => { + stream.callbacks.onResponse({ + id: 'subscription-event', + ok: true, + result: { type: 'snapshots', snapshots: [chatSnapshot()], authoritative: true }, + _meta: { runtimeId: 'runtime-a' } + }) + await settle() + }) + + expect(mirroredChats().map((tab) => tab.entityId)).toEqual(['codex-1']) + // Nothing retried the listing: the stream alone carried the tabs. + expect( + runtimeCall.mock.calls.filter(([request]) => request.method === 'session.tabs.listAll') + ).toHaveLength(1) + }) +}) From eeb2b39c4f4546fee0e0528287f323936943d392 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:03:52 -0700 Subject: [PATCH 075/107] test(mobile): a slow host's timed-out first tab list still ends with the host's tabs from the stream --- ...mobile-session-tabs-reconciliation.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts b/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts index 3b4199948487..272b00ca5857 100644 --- a/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts +++ b/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts @@ -191,6 +191,33 @@ describe('useMobileSessionTabsReconciliation', () => { vi.useRealTimers() }) + // A host's first answer waits on its startup work; on the first boot after an upgrade that can + // outlast the list's own deadline. The tabs stream, already open, still brings the host's tabs. + it("ends with the host's tabs from the stream when a slow host's first list times out", async () => { + sendRequest.mockImplementationOnce(() => + Promise.reject(new Error('Request timed out: session.tabs.list')) + ) + await mount() + expect(subscribe).toHaveBeenCalledWith( + 'session.tabs.subscribe', + { worktree: 'id:repo::worktree' }, + expect.any(Function) + ) + expect(sendRequest).toHaveBeenCalledTimes(1) + expect(applySessionTabs).not.toHaveBeenCalled() + + // The host finishes its startup work: the stream's first frame is the worktree's whole snapshot. + await emitStream({ type: 'snapshot', snapshotVersion: 7, tabs: ['chat-1'] }) + + expect(consumeAcceptedSessionTabs).toHaveBeenCalledWith( + expect.objectContaining({ tabs: ['chat-1'] }), + ['chat-1'], + 'stream' + ) + // The snapshot also re-arms a list, which now answers at once. + expect(sendRequest).toHaveBeenCalledTimes(2) + }) + it('runs one terminal health sweep and zero tab lists in a certified warm minute', async () => { await mount() await emitStream({ type: 'updated', snapshotVersion: 1, tabs: ['tab-1'] }) From 8fc0bf3f976342e372e413060511e887be06a32f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 12:26:53 -0700 Subject: [PATCH 076/107] perf(native-chat): listed chats' missing statuses are derived before the tab listing answers On the first launch after the upgrade to stored status, the startup step now folds and writes the row of every listed chat with history here and no row, before the listing answers, at most 16 rows per unsynced transaction and yielding once a task has run 16 ms. The seed then publishes them from the same read, and the settle takes any a crash left with work in the same launch, so no status arrives after paint and no chat is shown from its rows before its settle. A launch where every listed chat has its row does no extra work. A failed catch-up is logged and startup goes on. Removed: the pass after the listing that derived rows, seeded them a slice at a time and stored unfinished ones unshown; its on-screen ordering; the startup isListed dep; the startup settleClosedChat member. The restore after the listing opens only chats still in a per-chat file, with a corrupt history, or with a row this build cannot read. --- docs/reference/agent-status-store.md | 6 +- .../journal-session-status-backfill.ts | 6 +- .../structured-agent-session-host.ts | 1 + .../structured-agent-session-reveal.ts | 10 +- ...ctured-agent-session-startup-state.test.ts | 44 +- .../structured-agent-session-startup-state.ts | 69 +-- ...nt-session-startup-status-catch-up.test.ts | 212 +++++++ ...d-agent-session-startup-status-catch-up.ts | 146 +++++ ...gent-session-startup-status-derive.test.ts | 185 ------- ...red-agent-session-startup-status-derive.ts | 171 ------ ...ured-agent-session-status-catch-up.test.ts | 317 +++++++++++ ...ctured-agent-session-status-derive.test.ts | 520 ------------------ ...tore-structured-agent-session-tabs-once.ts | 4 +- ...runtime-structured-session-restore.test.ts | 1 + ...ructured-agent-session-restoration.test.ts | 77 --- .../structured-agent-session-startup-step.ts | 9 +- ...ctured-agent-session-support-probe.test.ts | 1 + ...-agent-session-tab-listing-startup.test.ts | 28 +- ...structured-chat-startup-fill-order.test.ts | 99 ---- ...ved-on-screen-structured-agent-sessions.ts | 44 -- ...ed-session-reopened-tab-close.unit.test.ts | 1 + 21 files changed, 746 insertions(+), 1205 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts delete mode 100644 src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts delete mode 100644 src/shared/saved-on-screen-structured-agent-sessions.ts diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 116b2253b1e6..5e102624e579 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -133,9 +133,9 @@ Two rules the ingest must keep: there beside its journal (`journal_session_state`), written in the same transaction as the journal rows it describes, so it is always current. A chat last written before that table existed has no row at first: a listed - chat gets one at startup, computed from its journal rows without opening it - (or from its open, when those rows show work a gone process left); any - other chat gets one when it is opened. + chat gets one at startup, before the tab listing answers, computed from its + journal rows without opening it, so the republish and the settle below cover + it too; any other chat gets one when it is opened, which also settles it. At host startup the host republishes a settled chat's row into the store without opening the chat, and settles a chat a gone process left with work first. A structured row in `last-status.json` would diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index a78d8992c3b1..9c49f47fa894 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -21,11 +21,11 @@ import { /** Row JSON per part, in UTF-16 units: a page of large rows is split, so each main-thread task * handles about the same bytes (a quarter of a full page of the seed's 2.3 KB rows). */ -export const IMPORT_BATCH_CHARS = 256 * 1024 +const IMPORT_BATCH_CHARS = 256 * 1024 /** Each batch split so no part holds more than `maxChars` of row JSON; a larger row is a part alone. * Only a batch's last part keeps its `last`. */ -export function* charBoundedBatches( +function* charBoundedBatches( batches: Iterable<{ rows: Row[]; last: boolean }>, maxChars = IMPORT_BATCH_CHARS ): Generator<{ rows: Row[]; last: boolean }> { @@ -57,7 +57,7 @@ export type FoldedJournalSessionStatus = { type FoldOptions = { batchRows?: number batchChars?: number - /** Ends each part's task: the next macrotask by default; a background job paces here. */ + /** Ends each part's task: the next macrotask by default; a caller can yield only when due. */ yieldTask?: () => Promise /** Quit: the fold stops within one part and nothing is written. */ signal?: AbortSignal diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 11d76bf34678..caef9bb85be9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -238,6 +238,7 @@ export class StructuredAgentSessionHost { restoreReadableSessions = (ids?: readonly string[]) => this.restore.restoreReadableSessions(ids) // Startup, from each chat's stored state: see `structured-agent-session-startup-state`. + catchUpMissingStatuses = (ids: readonly string[]) => this.restore.catchUpMissingStatuses(ids) seedStoredStatuses = (ids: readonly string[]) => this.restore.seedStoredStatuses(ids) settleOwedSessions = (ids: readonly string[]) => this.restore.settleOwedSessions(ids) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index df9fca150dbf..ea2e1615712e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -124,20 +124,14 @@ export function createStructuredAgentSessionHostRestore( recoveryBudgetMs: deps.startupRecoveryBudgetMs, serialize: rest.serialize, hasSession: rest.hasSession, - isListed: readRestore.isListed, isDisposed: rest.isDisposed }) return { reconcileRestartLeases: async () => { await reconcile('startup') }, - // The listed chats the tab list left to it: rows derived without an open first, then the rest. - restoreReadableSessions: (sessionIds) => - gate.run(async () => - restorer.restore( - sessionIds === undefined ? undefined : await startup.deriveMissingStatuses(sessionIds) - ) - ), + // The listed chats the tab list left to it: still in a per-chat file, corrupt, or unreadable. + restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), ...startup } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index bdab9bba5ffe..411b9f410f5b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -83,6 +83,7 @@ async function crashMidTurn(rig: RestTestRig, sessionId: string): Promise /** What startup runs, in order, on the ids the tab list names. */ async function startup(rig: RestTestRig, listed: readonly string[]) { await rig.host.reconcileRestartLeases() + await rig.host.catchUpMissingStatuses(listed) const background = rig.host.seedStoredStatuses(listed) await rig.host.settleOwedSessions(listed) await rig.host.restoreReadableSessions(background) @@ -319,7 +320,8 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(opened(rig, [...listed, ...unlisted]).toSorted()).toEqual( ['session-crashed', ...unlisted, 'session-uncopied'].toSorted() ) - expect(background.toSorted()).toEqual(['session-rowless', 'session-uncopied'].toSorted()) + // Its row is derived before the seed, so only the per-chat file is left to the restore. + expect(background).toEqual(['session-uncopied']) expect(latestRestTestStatus(rig, 'session-draft')).toMatchObject({ status: 'idle' }) // Settled with no user action; the listed one is open and never showed its pre-crash work. for (const sessionId of ['session-crashed', ...unlisted]) { @@ -342,7 +344,7 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) } - // The missing status was written back, and its row published, with no open. + // The missing status was written back before the seed, and its row published, with no open. expect(readTestJournalSessionStatus(rig.root, 'session-rowless')).toMatchObject({ lifecycle: 'idle' }) @@ -616,7 +618,6 @@ describe('a recovery that never answers', () => { restoreListed, serialize: (_sessionId, task) => task(), hasSession: () => false, - isListed: () => true, isDisposed: () => false, recoveryBudgetMs: 20 }) @@ -674,43 +675,6 @@ describe('a listed chat whose recovery never answers', () => { }) describe('a stored status no settle here can clear (R2A-4)', () => { - it('is refused by the shared closed-chat settle, which opens nothing for it', async () => { - const opens: string[] = [] - const openDeps = new Proxy( - {}, - { - get: (_target, key) => { - opens.push(String(key)) - throw new Error('the settle opened a chat this host cannot settle') - } - } - ) - const refused: AgentSessionRecord[] = [] - const canSettle = (record: AgentSessionRecord | null): record is AgentSessionRecord => { - if (record) { - refused.push(record) - } - return false - } - const state = createStructuredAgentSessionStartupState({ - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a refused settle reads no open dependency; any read throws and fails the test. - openDeps: openDeps as StructuredAgentSessionStartupStateDeps['openDeps'], - canSettle, - seedStatus: vi.fn(), - resolveRecovery: vi.fn(async () => true), - restoreListed: vi.fn(async () => undefined), - serialize: (_sessionId, task) => task(), - hasSession: () => false, - isListed: () => true, - isDisposed: () => false - }) - const record = agentSessionRecordFixture() - - await expect(state.settleClosedChat(record)).resolves.toBe(false) - expect(refused).toEqual([record]) - expect(opens).toEqual([]) - }) - it('drops the row of a chat whose record is gone or whose provider this host does not serve, so the next boot selects neither', async () => { const rig = await newRig() await crashMidSend(rig, 'session-gone', false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index bdb8846fdea0..6b0547566c0f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -1,6 +1,7 @@ // What host startup does with each chat's stored status, before any client lists a tab. // -// Seed: every settled, listed chat's status row is published from its stored status, so session +// Catch-up: a listed chat with history here and no stored status yet (the first launch after the +// upgrade) gets its row from its rows alone, so what follows reads stored status for it too. Seed: every settled, listed chat's status row is published from its stored status, so session // lists have it at paint, without opening the chat. Settle: first every lease a crash left // `recovering` is resolved (a surviving provider process is stopped and its death recorded), so // each verdict below is taken once, from that evidence. Then every chat whose stored status shows @@ -8,8 +9,8 @@ // settlement plan. A listed one goes through the restart restore's own per-chat worker and stays // open; any other is settled and closed, never indexed or published. Chat commands wait for the // settle (see `StructuredAgentSessionHostDeps.commandsReady`); listing, paint and status reads do -// not. A listed chat with no stored status yet (an older build wrote it last, or a per-chat file is -// not yet copied) is left to the background restore after the listing, which is the same worker. +// not. A listed chat whose history is still in a per-chat file, or is corrupt, is left to the +// background restore after the listing, which is the same worker. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' @@ -21,14 +22,15 @@ import { deleteJournalSessionStatus, isUnsettledJournalSessionStatus, readJournalSessionStatuses, - readUnsettledJournalSessionIds + readUnsettledJournalSessionIds, + type StoredJournalSessionStatus } from '../agent-session-journal/journal-session-state' import { openStructuredAgentSessionConversationJournal, type StructuredAgentSessionConversationOpenDeps } from './structured-agent-session-conversation-open' import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' -import { deriveMissingStatuses } from './structured-agent-session-startup-status-derive' +import { catchUpMissingStatuses } from './structured-agent-session-startup-status-catch-up' // Record-only (a probe and at most a process stop each), so a few at once. const RECOVERY_CONCURRENCY = 4 @@ -60,62 +62,67 @@ export type StructuredAgentSessionStartupStateDeps = { recoveryBudgetMs?: number serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean - /** Whether the chat still has its tab. */ - isListed: (sessionId: string) => boolean isDisposed: () => boolean } export type StructuredAgentSessionStartupState = { + /** Writes the stored status of every listed chat with history here and none yet, before the + * listing answers. Never rejects. See `catchUpMissingStatuses`. */ + catchUpMissingStatuses: (listedIds: readonly string[]) => Promise /** Seeds settled listed chats; answers the listed ids the background restore still opens. */ seedStoredStatuses: (listedIds: readonly string[]) => string[] /** Settles every chat a gone process left with work, once per host. Never rejects. */ settleOwedSessions: (listedIds: readonly string[]) => Promise - /** - * The one settle of a chat nothing holds open. Call it inside the chat's serialize; it never - * waits on the startup gate. True when it opened and closed the chat, whether or not the chat - * had work (so a settled chat is opened again). False, opening nothing, when `canSettle` rejects - * the chat, the chat is already open, or after quit. Rejects when the open or the close throws, - * so callers must catch. - */ - settleClosedChat: (record: AgentSessionRecord) => Promise - /** - * The background pass's first step: each listed chat with history here and no status row gets - * its row from its rows alone, opening nothing, folded one chat at a time in the order given (the - * startup step puts the chats on screen first) and written a slice at a time. A row showing work - * a gone process left is stored but never shown; a corrupt history gets no row, so its open and - * rebuild run. Answers the chats still to open, which their open settles, rebuilds or gives a - * row; see `deriveMissingStatuses`. - */ - deriveMissingStatuses: (sessionIds: readonly string[]) => Promise + /** Startup's own chat work: true from the catch-up's start until the settle has ended. */ + isSettling: () => boolean } export function createStructuredAgentSessionStartupState( deps: StructuredAgentSessionStartupStateDeps ): StructuredAgentSessionStartupState { + let catchingUp = 0 + // What the catch-up read for a listing, which that listing's seed reads instead of reading again. + let caughtUp: { listedIds: readonly string[]; stored: StoredJournalSessionStatus[] } | null = null let settling: Promise | null = null + let settled = false return { - seedStoredStatuses: (listedIds) => seedStoredStatuses(deps, listedIds), + catchUpMissingStatuses: async (listedIds) => { + catchingUp += 1 + try { + const stored = await catchUpMissingStatuses(deps, listedIds) + caughtUp = stored ? { listedIds, stored } : null + } finally { + catchingUp -= 1 + } + }, + seedStoredStatuses: (listedIds) => { + const read = caughtUp?.listedIds === listedIds ? caughtUp.stored : null + caughtUp = null + return seedStoredStatuses(deps, listedIds, read) + }, settleOwedSessions: (listedIds) => { - settling ??= settleOwedSessions(deps, listedIds) + settling ??= settleOwedSessions(deps, listedIds).finally(() => { + settled = true + }) return settling }, - settleClosedChat: (record) => settleClosed(deps, record), - deriveMissingStatuses: (sessionIds) => deriveMissingStatuses(deps, sessionIds) + isSettling: () => catchingUp > 0 || (settling !== null && !settled) } } function seedStoredStatuses( deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[] + listedIds: readonly string[], + read?: readonly StoredJournalSessionStatus[] | null ): string[] { const database = deps.openDeps.journalDatabase // A newer build's database: nothing is stored this build can read, and every chat reads as it does. if (database.readOnly) { return [...listedIds] } - let stored: ReturnType + let stored: readonly StoredJournalSessionStatus[] try { - stored = readJournalSessionStatuses(database.db, listedIds) + stored = read ?? readJournalSessionStatuses(database.db, listedIds) } catch (error) { // Fails open: every listed chat is restored in the background, as before stored status existed. deps.openDeps.logger.warn('reading stored chat status failed', { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts new file mode 100644 index 000000000000..407204af22b4 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts @@ -0,0 +1,212 @@ +// The catch-up's tasks and its quit: a launch where every listed chat has its row does no work at +// all, a launch that derives rows yields only when a task has run long enough, and quit stops it +// with nothing more written, for the next launch to redo. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type * as TimersPromises from 'node:timers/promises' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' +import { + closeTestJournalHostDatabases, + createTrackedJournalOpener, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from '../agent-session-journal/journal-host-database-test-support' +import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' +import { + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CORPUS +} from '../agent-session-journal/journal-session-state-test-corpus' +import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' + +// What the catch-up did, in order: each yield to the event loop, and each fold it finished. +const trace = vi.hoisted(() => { + const log: { events: string[]; afterFold: ((sessionId: string) => void) | null } = { + events: [], + afterFold: null + } + return log +}) + +vi.mock('node:timers/promises', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + setImmediate: (...args: Parameters) => { + trace.events.push('yield') + return actual.setImmediate(...args) + } + } +}) + +vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + foldJournalSessionStatus: async ( + ...args: Parameters + ) => { + const folded = await actual.foldJournalSessionStatus(...args) + trace.events.push(`fold:${args[1]}`) + trace.afterFold?.(args[1]) + return folded + } + } +}) + +import { catchUpMissingStatuses } from './structured-agent-session-startup-status-catch-up' +import { createStructuredAgentSessionStartupState } from './structured-agent-session-startup-state' + +const journals = createTrackedJournalOpener() +let root: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-status-catch-up-')) + trace.events = [] + trace.afterFold = null +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +/** Chats with a settled history in the host's database; `rowless` drops their status rows. */ +async function chats(sessionIds: readonly string[], { rowless }: { rowless: boolean }) { + for (const sessionId of sessionIds) { + const journal = await journals.open({ + identity: { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + }, + stateDirectory: root, + mintEpoch: () => `epoch-${sessionId}`, + currentFence: () => CORPUS_FENCE + }) + await JOURNAL_SESSION_STATE_CORPUS.settled(journal) + } + await journals.closeAll() + if (rowless) { + openTestJournalHostDatabase(root).db.exec('DELETE FROM journal_session_state') + } +} + +function deps( + disposed: { value: boolean } = { value: false } +): StructuredAgentSessionStartupStateDeps { + const stand = { + openDeps: { + journalDatabase: openTestJournalHostDatabase(root), + store: { + getRecord: (sessionId: string): AgentSessionRecord => ({ + ...agentSessionRecordFixture(), + sessionId + }), + listRecords: () => [] + }, + logger: { warn: vi.fn(), error: vi.fn() } + }, + canSettle: (candidate: AgentSessionRecord | null): candidate is AgentSessionRecord => + candidate !== null, + seedStatus: vi.fn(), + resolveRecovery: async () => true, + restoreListed: async () => undefined, + serialize: (_sessionId: string, task: () => Promise) => task(), + hasSession: () => false, + isDisposed: () => disposed.value + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the catch-up reads only the database, records, logger and the members above. + return stand as unknown as StructuredAgentSessionStartupStateDeps +} + +const yields = () => trace.events.filter((event) => event === 'yield').length +const folds = () => trace.events.filter((event) => event.startsWith('fold:')) + +describe('the startup status catch-up', () => { + it('does nothing on a launch where every listed chat has its row', async () => { + const ids = Array.from({ length: 40 }, (_, index) => `session-${index}`) + await chats(ids, { rowless: false }) + const stand = deps() + const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') + trace.events = [] + + const stored = await catchUpMissingStatuses(stand, ids) + + // No fold, no yield, no transaction; the seed reads what this read. + expect(folds()).toEqual([]) + expect(yields()).toBe(0) + expect(transaction).not.toHaveBeenCalled() + expect(stored?.map(({ sessionId }) => sessionId).toSorted()).toEqual(ids.toSorted()) + }) + + it('yields only once a task has run long enough, not before every chat', async () => { + const ids = Array.from({ length: 30 }, (_, index) => `session-${index}`) + await chats(ids, { rowless: true }) + // Each clock read moves 3 ms: a task reaches the 16 ms budget every few chats. + let clock = 0 + vi.spyOn(performance, 'now').mockImplementation(() => (clock += 3)) + trace.events = [] + + const stored = await catchUpMissingStatuses(deps(), ids) + + expect(folds()).toHaveLength(ids.length) + expect(yields()).toBeGreaterThan(0) + expect(yields()).toBeLessThan(ids.length / 2) + expect(ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(true) + expect(stored?.every(({ status }) => status?.lifecycle === 'idle')).toBe(true) + }) + + it('writes nothing more after quit, and the next launch redoes what is left', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `session-${index}`) + await chats(ids, { rowless: true }) + const disposed = { value: false } + // Quit lands during the third fold; every task is due a yield, which sees it. + let clock = 0 + vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) + trace.afterFold = (sessionId) => { + if (sessionId === 'session-2') { + disposed.value = true + } + } + const stand = deps(disposed) + const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') + + expect(await catchUpMissingStatuses(stand, ids)).toBeNull() + + // Quit came before the first batch was due, so nothing was written at all. + expect(transaction).not.toHaveBeenCalled() + expect(ids.some((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(false) + + trace.afterFold = null + disposed.value = false + await catchUpMissingStatuses(deps(), ids) + expect(ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(true) + }) + + it("reads as startup's own chat work from the catch-up's start until the settle ends", async () => { + const ids = ['session-0', 'session-1'] + await chats(ids, { rowless: true }) + const state = createStructuredAgentSessionStartupState(deps()) + const during: boolean[] = [] + trace.afterFold = () => during.push(state.isSettling()) + expect(state.isSettling()).toBe(false) + + const catchingUp = state.catchUpMissingStatuses(ids) + expect(state.isSettling()).toBe(true) + await catchingUp + expect(during).toEqual([true, true]) + const settling = state.settleOwedSessions(ids) + expect(state.isSettling()).toBe(true) + await settling + + expect(state.isSettling()).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts new file mode 100644 index 000000000000..99ffa2b47ada --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts @@ -0,0 +1,146 @@ +// The first launch after the upgrade to stored status: listed chats with history here and no status +// row get their rows from their rows alone, before the tab listing answers. The seed that follows +// then publishes them, and the settle takes any a gone process left with work, as on every launch. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import { + foldJournalSessionStatus, + writeJournalSessionStatuses, + type FoldedJournalSessionStatus +} from '../agent-session-journal/journal-session-status-backfill' +import { + readJournalSessionStatuses, + type StoredJournalSessionStatus +} from '../agent-session-journal/journal-session-state' +import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' + +// Rows per transaction: one commit per chat rewrites the same table and index pages each time. +const WRITE_BATCH_CHATS = 16 +// Work per main-process task before yielding, so window IPC and the hook server keep answering. +const TASK_BUDGET_MS = 16 + +/** + * Folds and writes the row of every listed chat that has history here and none yet. Answers the + * listed chats' stored statuses, the written ones included, for the seed to publish without reading + * them again, or null when it read nothing (a read-only database, quit, or a failure, which it + * reports). Rows that changed meanwhile belong to chats opened meanwhile, which the seed skips. + * Never rejects: the seed, the settle and the listing go on whatever happens here. + */ +export async function catchUpMissingStatuses( + deps: StructuredAgentSessionStartupStateDeps, + listedIds: readonly string[] +): Promise { + const database = deps.openDeps.journalDatabase + if (database.readOnly) { + return null + } + try { + const stored = readJournalSessionStatuses(database.db, listedIds) + const missing = stored.flatMap(({ sessionId, status }) => (status ? [] : [sessionId])) + // Every listed chat has its row: no fold, no yield, no transaction. + if (missing.length === 0) { + return stored + } + const written = await foldAndWrite(deps, missing) + if (!written) { + return null + } + return stored.map((row) => { + const status = written.get(row.sessionId) + return status ? { ...row, status } : row + }) + } catch (error) { + deps.openDeps.logger.warn('deriving missing chat statuses at startup failed', { + scope: 'startup-status-catch-up', + error + }) + return null + } +} + +/** The written chats' statuses by id, or null after quit, which stops a fold within one part and + * writes nothing more. */ +async function foldAndWrite( + deps: StructuredAgentSessionStartupStateDeps, + sessionIds: readonly string[] +): Promise | null> { + const quit = new AbortController() + let taskStart = performance.now() + const yieldWhenDue = async (): Promise => { + if (performance.now() - taskStart < TASK_BUDGET_MS) { + return + } + await yieldToEventLoop() + taskStart = performance.now() + if (deps.isDisposed()) { + quit.abort() + } + } + const written = new Map() + let batch: FoldedJournalSessionStatus[] = [] + const flush = (): void => { + for (const folded of writeBatch(deps, batch)) { + written.set(folded.sessionId, folded.status) + } + batch = [] + } + for (const sessionId of sessionIds) { + await yieldWhenDue() + if (quit.signal.aborted || deps.isDisposed()) { + return null + } + const record = deps.openDeps.store.getRecord(sessionId) + if (!record || !deps.canSettle(record) || deps.hasSession(sessionId)) { + continue + } + const folded = await foldOne(deps, sessionId, { + yieldTask: yieldWhenDue, + signal: quit.signal + }) + // A corrupt history gets no row, so every launch opens it until its open rebuilds it. + if (folded && !folded.load.corrupt) { + batch.push(folded) + } + if (batch.length >= WRITE_BATCH_CHATS) { + flush() + } + } + if (quit.signal.aborted || deps.isDisposed()) { + return null + } + flush() + return written +} + +async function foldOne( + deps: StructuredAgentSessionStartupStateDeps, + sessionId: string, + options: { yieldTask: () => Promise; signal: AbortSignal } +): Promise { + try { + return await foldJournalSessionStatus(deps.openDeps.journalDatabase, sessionId, options) + } catch (error) { + deps.openDeps.logger.warn('deriving a chat status from its rows failed', { + scope: 'startup-status-catch-up', + sessionId, + error + }) + return null + } +} + +/** One transaction; a chat that moved or got a row since its fold is skipped and left rowless. */ +function writeBatch( + deps: StructuredAgentSessionStartupStateDeps, + batch: FoldedJournalSessionStatus[] +): FoldedJournalSessionStatus[] { + try { + return writeJournalSessionStatuses(deps.openDeps.journalDatabase, batch) + } catch (error) { + deps.openDeps.logger.warn('writing chat statuses derived from their rows failed', { + scope: 'startup-status-catch-up', + error + }) + return [] + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts deleted file mode 100644 index 9a367c895499..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -// The startup status pass's tasks: a chat with nothing to fold (no history in the host's database -// yet, or a row already) goes by without a task of its own, a long run of them still yields, and a -// chat that folds always starts a task. - -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import type * as TimersPromises from 'node:timers/promises' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' -import { - closeTestJournalHostDatabases, - createTrackedJournalOpener, - openTestJournalHostDatabase, - readTestJournalSessionStatus -} from '../agent-session-journal/journal-host-database-test-support' -import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' -import { - CORPUS_FENCE, - JOURNAL_SESSION_STATE_CORPUS -} from '../agent-session-journal/journal-session-state-test-corpus' -import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' - -// What the pass did, in order: each yield to the event loop, and each fold it started. -const trace = vi.hoisted(() => { - const log: { events: string[] } = { events: [] } - return log -}) - -vi.mock('node:timers/promises', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - setImmediate: (...args: Parameters) => { - trace.events.push('yield') - return actual.setImmediate(...args) - } - } -}) - -vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - foldJournalSessionStatus: (...args: Parameters) => { - trace.events.push(`fold:${args[1]}`) - return actual.foldJournalSessionStatus(...args) - } - } -}) - -import { deriveMissingStatuses } from './structured-agent-session-startup-status-derive' - -const journals = createTrackedJournalOpener() -let root: string - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'orca-startup-status-derive-')) -}) - -afterEach(async () => { - vi.restoreAllMocks() - await journals.closeAll() - closeTestJournalHostDatabases() - await rm(root, { recursive: true, force: true }) -}) - -function record(sessionId: string): AgentSessionRecord { - return { ...agentSessionRecordFixture(), sessionId } -} - -/** Chats with a settled history in the host's database and no status row: each one folds. */ -async function chatsWithHistory(sessionIds: readonly string[]): Promise { - for (const sessionId of sessionIds) { - const journal = await journals.open({ - identity: { - sessionId, - workspaceId: 'ws-1', - hostId: 'local', - agent: 'codex', - providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } - }, - stateDirectory: root, - mintEpoch: () => `epoch-${sessionId}`, - currentFence: () => CORPUS_FENCE - }) - await JOURNAL_SESSION_STATE_CORPUS.settled(journal) - } - await journals.closeAll() - openTestJournalHostDatabase(root).db.exec('DELETE FROM journal_session_state') -} - -function deps(seeded: string[] = []): StructuredAgentSessionStartupStateDeps { - const stand = { - openDeps: { - journalDatabase: openTestJournalHostDatabase(root), - store: { getRecord: record, listRecords: () => [] }, - logger: { warn: vi.fn(), error: vi.fn() } - }, - canSettle: (candidate: AgentSessionRecord | null): candidate is AgentSessionRecord => - candidate !== null, - seedStatus: (chat: AgentSessionRecord) => { - seeded.push(chat.sessionId) - }, - resolveRecovery: async () => true, - restoreListed: async () => undefined, - serialize: (_sessionId: string, task: () => Promise) => task(), - hasSession: () => false, - isListed: () => true, - isDisposed: () => false - } - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pass reads only the database, records, logger and the members above. - return stand as unknown as StructuredAgentSessionStartupStateDeps -} - -const yields = () => trace.events.filter((event) => event === 'yield').length - -describe('the startup status pass over chats with nothing to fold', () => { - it('passes them all, in order, without a task each and without a write', async () => { - const ids = Array.from({ length: 100 }, (_, index) => `session-file-${index}`) - const stand = deps() - const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') - trace.events = [] - - const toOpen = await deriveMissingStatuses(stand, ids) - - expect(toOpen).toEqual(ids) - expect(trace.events.filter((event) => event.startsWith('fold:'))).toEqual([]) - expect(yields()).toBeLessThan(ids.length / 4) - // An empty slice commits nothing. - expect(transaction).not.toHaveBeenCalled() - }) - - it('still yields during a long run of them', async () => { - const ids = Array.from({ length: 60 }, (_, index) => `session-file-${index}`) - const stand = deps() - // Each clock read moves 3 ms: a run of skips crosses the 8 ms bound every few chats. - let clock = 0 - vi.spyOn(performance, 'now').mockImplementation(() => (clock += 3)) - trace.events = [] - - expect(await deriveMissingStatuses(stand, ids)).toEqual(ids) - - expect(yields()).toBeGreaterThanOrEqual(10) - expect(yields()).toBeLessThan(ids.length) - }) - - it('keeps the order in a mixed pass and starts a task before each chat that folds', async () => { - await chatsWithHistory(['session-fold-1', 'session-fold-2']) - const ids = [ - 'session-file-1', - 'session-fold-1', - 'session-file-2', - 'session-file-3', - 'session-fold-2', - 'session-file-4' - ] - const seeded: string[] = [] - const stand = deps(seeded) - trace.events = [] - - const toOpen = await deriveMissingStatuses(stand, ids) - - expect(toOpen).toEqual(['session-file-1', 'session-file-2', 'session-file-3', 'session-file-4']) - expect(seeded).toEqual(['session-fold-1', 'session-fold-2']) - for (const sessionId of ['session-fold-1', 'session-fold-2']) { - expect(readTestJournalSessionStatus(root, sessionId)).toMatchObject({ lifecycle: 'idle' }) - } - // A yield between the start (or the previous fold) and each fold. - let sinceFold: string[] = [] - for (const event of trace.events) { - if (event.startsWith('fold:')) { - expect(sinceFold).toContain('yield') - sinceFold = [] - } else { - sinceFold.push(event) - } - } - expect(trace.events.filter((event) => event.startsWith('fold:'))).toEqual([ - 'fold:session-fold-1', - 'fold:session-fold-2' - ]) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts deleted file mode 100644 index b295b03d1310..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-derive.ts +++ /dev/null @@ -1,171 +0,0 @@ -// The background pass's first step after an upgrade: listed chats with history here and no status -// row get their rows from their rows alone, without an open. Chats fold one per task; their rows are -// written a slice at a time, in one transaction per slice, so the pass appends few WAL pages. - -import { setImmediate as yieldToEventLoop } from 'node:timers/promises' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { - foldableJournalSessionEpoch, - foldJournalSessionStatus, - writeJournalSessionStatuses, - type FoldedJournalSessionStatus -} from '../agent-session-journal/journal-session-status-backfill' -import { isUnsettledJournalSessionStatus } from '../agent-session-journal/journal-session-state' -import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' - -// A write per slice, not per chat: each per-chat commit rewrites the same table and index pages. -const SLICE_CHATS = 16 -const SLICE_MS = 50 -// Chats with nothing to fold go by without a yield each; a run of them still yields once this long. -const SKIP_RUN_MS = 8 - -/** `shown` false: an unfinished chat's row, stored so the next boot selects the chat until its open - * settles it, but never displayed from rows. */ -type Folded = FoldedJournalSessionStatus & { record: AgentSessionRecord; shown: boolean } - -/** Answers the chats still to open: the chats it could not fold, those with a corrupt history - * (given no row, so every boot opens one until its rebuild), and those whose rows show work a gone - * process left, which their open settles, in the order given; then, after each slice's write, the - * chats of that slice the write had to skip. */ -export async function deriveMissingStatuses( - deps: StructuredAgentSessionStartupStateDeps, - sessionIds: readonly string[] -): Promise { - if (deps.openDeps.journalDatabase.readOnly) { - return [...sessionIds] - } - // Quit stops a fold within one part, and nothing more is written. - const quit = new AbortController() - const yieldTask = async () => { - await yieldToEventLoop() - if (deps.isDisposed()) { - quit.abort() - } - } - const toOpen: string[] = [] - let slice: Folded[] = [] - let sliceStart = 0 - let lastYield = performance.now() - for (const sessionId of sessionIds) { - const record = deps.openDeps.store.getRecord(sessionId) - if (!record || !canFold(deps, record)) { - if (performance.now() - lastYield >= SKIP_RUN_MS) { - await yieldTask() - lastYield = performance.now() - } - if (deps.isDisposed()) { - return [] - } - toOpen.push(sessionId) - continue - } - // A task per chat that folds: short chats fold in one part, and a pass of them must not be one task. - await yieldTask() - lastYield = performance.now() - if (deps.isDisposed()) { - return [] - } - // Opened during the yield: its open writes and publishes its row. - const folded = deps.hasSession(sessionId) - ? null - : await foldFromRows(deps, record, { yieldTask, signal: quit.signal }) - if (!folded || !folded.shown) { - toOpen.push(sessionId) - } - if (!folded) { - continue - } - if (slice.length === 0) { - sliceStart = performance.now() - } - slice.push(folded) - if (slice.length >= SLICE_CHATS || performance.now() - sliceStart >= SLICE_MS) { - toOpen.push(...writeSlice(deps, slice)) - slice = [] - } - } - if (deps.isDisposed()) { - return [] - } - toOpen.push(...writeSlice(deps, slice)) - return toOpen -} - -/** Whether the chat has rows to fold here, decided without a task of its own. A check that throws - * answers yes, so the fold reports the failure. */ -function canFold( - deps: StructuredAgentSessionStartupStateDeps, - record: AgentSessionRecord -): boolean { - if (!deps.canSettle(record) || deps.hasSession(record.sessionId)) { - return false - } - try { - return foldableJournalSessionEpoch(deps.openDeps.journalDatabase, record.sessionId) !== null - } catch { - return true - } -} - -/** A chat's status from its rows; null leaves it to its open, rowless, as for a corrupt history: a - * stored row would let the next boot seed it and skip the open its rebuild needs. One with work a - * gone process left is still written, so startup selects it until its open settles it, but it is - * never shown: only its open publishes its status. */ -async function foldFromRows( - deps: StructuredAgentSessionStartupStateDeps, - record: AgentSessionRecord, - options: { yieldTask: () => Promise; signal: AbortSignal } -): Promise { - try { - const folded = await foldJournalSessionStatus( - deps.openDeps.journalDatabase, - record.sessionId, - options - ) - if (!folded || folded.load.corrupt) { - return null - } - return { ...folded, record, shown: !isUnsettledJournalSessionStatus(folded.status) } - } catch (error) { - deps.openDeps.logger.warn('deriving a chat status from its rows failed', { - scope: 'startup-status-derive', - sessionId: record.sessionId, - error - }) - return null - } -} - -/** Writes the slice's rows in one transaction and seeds each written chat still listed and closed; - * answers the shown chats left to their open (one a send or an open moved meanwhile, or all of them - * if the write failed). */ -function writeSlice(deps: StructuredAgentSessionStartupStateDeps, slice: Folded[]): string[] { - let written: FoldedJournalSessionStatus[] - try { - written = writeJournalSessionStatuses(deps.openDeps.journalDatabase, slice) - } catch (error) { - deps.openDeps.logger.warn('writing chat statuses derived from their rows failed', { - scope: 'startup-status-derive', - error - }) - return slice.flatMap(({ sessionId, shown }) => (shown ? [sessionId] : [])) - } - const writtenIds = new Set(written.map(({ sessionId }) => sessionId)) - for (const folded of slice) { - // A tab closed since the fold gets no sidebar row; one opened since publishes its own. - if ( - folded.shown && - writtenIds.has(folded.sessionId) && - deps.isListed(folded.sessionId) && - !deps.hasSession(folded.sessionId) - ) { - deps.seedStatus(folded.record, { - projected: folded.status.summary, - lastActivityAt: folded.status.lastActivityAt - }) - } - } - return slice.flatMap(({ sessionId, shown }) => - shown && !writtenIds.has(sessionId) ? [sessionId] : [] - ) -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts new file mode 100644 index 000000000000..6a5ba6e7d330 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts @@ -0,0 +1,317 @@ +// The first launch after the upgrade to stored status: listed chats with history and no status row +// get their rows from their rows alone before the tab listing answers, with no conversation opened. +// The seed then publishes them and the settle takes any a crash left with work, in the same launch. + +import { statSync } from 'node:fs' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as JournalSessionStateModule from '../agent-session-journal/journal-session-state' +import type * as JournalRecoveryModule from './agent-session-journal-recovery' +import { + closeTestJournalHostDatabases, + insertTestJournalRowJson, + liveTestJournalRows, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from '../agent-session-journal/journal-host-database-test-support' +import { + createRestTestRig, + restTestChat, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { + latestRestTestStatus, + restTestOpens +} from './structured-agent-session-rest-test-observations' + +// The listed chats' stored-status read fails once, when set: the catch-up's own read. +const reads = vi.hoisted(() => { + const control: { failNext: boolean } = { failNext: false } + return control +}) + +vi.mock('../agent-session-journal/journal-session-state', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + readJournalSessionStatuses: (...args: Parameters) => { + if (reads.failNext) { + reads.failNext = false + throw new Error('database is locked') + } + return actual.readJournalSessionStatuses(...args) + } + } +}) + +// Each chat an open found corrupt and sent through its rebuild. +const rebuilds = vi.hoisted(() => { + const opened: { sessionIds: string[] } = { sessionIds: [] } + return opened +}) + +vi.mock('./agent-session-journal-recovery', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + openAgentSessionJournalWithRecovery: async ( + ...args: Parameters + ) => { + const opened = await actual.openAgentSessionJournalWithRecovery(...args) + if (opened.recovery?.trigger === 'journal_corrupt') { + rebuilds.sessionIds.push(args[0].identity.sessionId) + } + return opened + } + } +}) + +const rigs: RestTestRig[] = [] + +afterEach(async () => { + reads.failNext = false + rebuilds.sessionIds.length = 0 + for (const rig of rigs.splice(0)) { + await rig.dispose() + } + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +async function newRig(): Promise { + const rig = await createRestTestRig() + rigs.push(rig) + return rig +} + +/** Every chat's row dropped, as version 5 finds a database an older build wrote. */ +function upgradeToEmptyStatusTable(rig: RestTestRig): void { + openTestJournalHostDatabase(rig.root).db.prepare('DELETE FROM journal_session_state').run() +} + +const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds + +/** Startup as the host's step runs it, through the settle; answers the restore's list. */ +async function startup(rig: RestTestRig): Promise { + const listed = listedIds(rig) + await rig.host.reconcileRestartLeases() + await rig.host.catchUpMissingStatuses(listed) + const background = rig.host.seedStoredStatuses(listed) + await rig.host.settleOwedSessions(listed) + return background +} + +/** A chat whose turn was still running when Orca died, with or without a tab. */ +async function crashMidTurn(rig: RestTestRig, sessionId: string, listed = true): Promise { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) + const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value + await rig.host + .collaboratorsForTests() + .sessions.get(sessionId)! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } + ) +} + +const firstStatus = (rig: RestTestRig, sessionId: string) => + rig.statusEvents.findIndex( + (event) => event.type === 'status' && event.session.sessionId === sessionId + ) + +/** The status stream's length when each chat first opened. */ +function recordOpens(rig: RestTestRig): Map { + const openedAt = new Map() + rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { + if (!openedAt.has(sessionId)) { + openedAt.set(sessionId, rig.statusEvents.length) + } + return null + }) + return openedAt +} + +describe('listed chats with no stored status after the upgrade', () => { + it('get their rows before the seed, from their rows alone, with no chat opened', async () => { + const rig = await newRig() + const ids = Array.from({ length: 12 }, (_, index) => `session-${index}`) + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + + const background = await startup(rig) + + expect(background).toEqual([]) + for (const sessionId of ids) { + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) + expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) + expect(restTestOpens(rig, sessionId)).toBe(0) + } + // Published by the seed, in the listing's order. + const order = ids.map((sessionId) => firstStatus(rig, sessionId)) + expect(order).toEqual(order.toSorted((a, b) => a - b)) + }) + + it('settles an unfinished one in the same launch, before commands go, never showing it from its rows', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-fine', { message: 'done' }) + await crashMidTurn(rig, 'session-running') + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + const listed = listedIds(rig) + const openedAt = recordOpens(rig) + await rig.host.reconcileRestartLeases() + + await rig.host.catchUpMissingStatuses(listed) + // Stored, so the settle selects it; not shown: the seed leaves an unsettled row to the settle. + expect(readTestJournalSessionStatus(rig.root, 'session-running')).toMatchObject({ + lifecycle: 'running' + }) + rig.host.seedStoredStatuses(listed) + expect(latestRestTestStatus(rig, 'session-running')).toBeUndefined() + // The settle is what chat commands wait on. + await rig.host.settleOwedSessions(listed) + + expect(restTestOpens(rig, 'session-fine')).toBe(0) + expect(openedAt.has('session-running')).toBe(true) + expect(firstStatus(rig, 'session-running')).toBeGreaterThanOrEqual( + openedAt.get('session-running')! + ) + expect(readTestJournalSessionStatus(rig.root, 'session-running')).toMatchObject({ + lifecycle: 'idle' + }) + expect(latestRestTestStatus(rig, 'session-running')?.status).not.toBe('working') + }) + + it('leaves an unlisted crashed chat with no row showing nothing until its open settles it', async () => { + const rig = await newRig() + await crashMidTurn(rig, 'session-closed', false) + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + expect(listedIds(rig)).not.toContain('session-closed') + + await startup(rig) + + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toBeNull() + expect(latestRestTestStatus(rig, 'session-closed')).toBeUndefined() + expect(restTestOpens(rig, 'session-closed')).toBe(0) + + // The first read opens it, and the open settles it. + await rig.host.history({ sessionId: 'session-closed', direction: 'tail' }) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle' + }) + expect(latestRestTestStatus(rig, 'session-closed')?.status).not.toBe('working') + }) + + it('appends few WAL pages for 247 chats: their rows are written 16 to a transaction', async () => { + const rig = await newRig() + const ids = Array.from({ length: 247 }, (_, index) => `session-page-${index}`) + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + const { db } = openTestJournalHostDatabase(rig.root) + // An empty WAL, so its size after the catch-up is what the catch-up appended. + db.pragma('wal_checkpoint(TRUNCATE)') + const pageSize = Number(db.pragma('page_size', { simple: true })) + + await rig.host.catchUpMissingStatuses(listedIds(rig)) + + const frames = + (statSync(join(rig.root, 'agent-session-journal.db-wal')).size - 32) / (pageSize + 24) + expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) + expect(ids.every((sessionId) => restTestOpens(rig, sessionId) === 0)).toBe(true) + // A commit per chat appends about 2.5 pages each, over 600 for these. + expect(frames).toBeLessThanOrEqual(150) + }, 120_000) + + it('gives a corrupt one no row, so the restore after the listing opens it and its rebuild runs', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-corrupt', { message: 'asked' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + const { db } = openTestJournalHostDatabase(rig.root) + const tip = liveTestJournalRows(db, 'session-corrupt').at(-1)! + insertTestJournalRowJson(db, 'session-corrupt', tip.seq + 1, '{') + upgradeToEmptyStatusTable(rig) + expect(loadTestJournal(rig.root, 'session-corrupt')).toMatchObject({ corrupt: true }) + await rig.boot() + + const background = await startup(rig) + + expect(readTestJournalSessionStatus(rig.root, 'session-corrupt')).toBeNull() + expect(latestRestTestStatus(rig, 'session-corrupt')).toBeUndefined() + expect(background).toEqual(['session-corrupt']) + await rig.host.restoreReadableSessions(background) + expect(rebuilds.sessionIds).toEqual(['session-corrupt']) + }) + + it('derives a row again on the next launch when its unsynced commit was lost', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-settled', { message: 'asked session-settled' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + await startup(rig) + const first = readTestJournalSessionStatus(rig.root, 'session-settled') + const firstPublished = latestRestTestStatus(rig, 'session-settled') + expect(first).toMatchObject({ lifecycle: 'idle' }) + + // A power cut loses the unsynced commit: the next launch finds no row. + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + const before = rig.statusEvents.length + await startup(rig) + + expect(readTestJournalSessionStatus(rig.root, 'session-settled')).toEqual(first) + expect(latestRestTestStatus(rig, 'session-settled')).toEqual(firstPublished) + expect( + rig.statusEvents + .slice(before) + .every((event) => event.type !== 'status' || event.session.sessionId === 'session-settled') + ).toBe(true) + expect(restTestOpens(rig, 'session-settled')).toBe(0) + }) + + it('lets the seed, the settle and the rest go on when the catch-up fails', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-rowed', { message: 'asked session-rowed' }) + await restTestChat(rig, 'session-rowless', { message: 'asked session-rowless' }) + await crashMidTurn(rig, 'session-crashed') + await rig.host.flushAllStreamedEvents() + await rig.crash() + openTestJournalHostDatabase(rig.root) + .db.prepare('DELETE FROM journal_session_state WHERE session_id = ?') + .run('session-rowless') + await rig.boot() + const warn = vi.spyOn(rig.host.deps.logger, 'warn') + reads.failNext = true + + const background = await startup(rig) + + expect(warn).toHaveBeenCalledWith( + 'deriving missing chat statuses at startup failed', + expect.objectContaining({ scope: 'startup-status-catch-up' }) + ) + // The seed read the rows itself, the rowless chat went to the restore, the settle ran. + expect(latestRestTestStatus(rig, 'session-rowed')).toMatchObject({ status: 'idle' }) + expect(background).toEqual(['session-rowless']) + expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ + lifecycle: 'idle' + }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts deleted file mode 100644 index dd815b248938..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-derive.test.ts +++ /dev/null @@ -1,520 +0,0 @@ -// The first boot after upgrading: listed chats with history and no status row get their rows from -// their rows alone, folded one at a time in the order given and written a slice at a time, with no -// conversation opened; a chat a crash left with work is still opened, so it is settled; and a send -// during the pass is not queued behind it. - -import { statSync } from 'node:fs' -import { join } from 'node:path' -import { afterEach, describe, expect, it, vi } from 'vitest' -import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION } from '../../../shared/agent-session-journal-types' -import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' -import type * as JournalRecoveryModule from './agent-session-journal-recovery' -import { - closeTestJournalHostDatabases, - insertTestJournalRow, - insertTestJournalRowJson, - liveTestJournalRows, - loadTestJournal, - openTestJournalHostDatabase, - readTestJournalSessionStatus -} from '../agent-session-journal/journal-host-database-test-support' -import { - createRestTestRig, - restTestChat, - sendRestTestMessage, - type RestTestRig -} from './structured-agent-session-rest-test-rig' -import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' -import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' -import { - latestRestTestStatus, - restTestOpens -} from './structured-agent-session-rest-test-observations' - -// The chats folded so far, whether or not their slice's rows are written yet. -const folds = vi.hoisted(() => { - const hooks: { - done: Set - /** Runs as each fold ends, before its slice is written. */ - after: ((sessionId: string) => Promise) | null - } = { done: new Set(), after: null } - return hooks -}) - -vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - foldJournalSessionStatus: async ( - ...args: Parameters - ) => { - const folded = await actual.foldJournalSessionStatus(...args) - folds.done.add(args[1]) - await folds.after?.(args[1]) - return folded - } - } -}) - -// Each chat an open found corrupt and sent through its rebuild. -const rebuilds = vi.hoisted(() => { - const opened: { sessionIds: string[] } = { sessionIds: [] } - return opened -}) - -vi.mock('./agent-session-journal-recovery', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - openAgentSessionJournalWithRecovery: async ( - ...args: Parameters - ) => { - const opened = await actual.openAgentSessionJournalWithRecovery(...args) - if (opened.recovery?.trigger === 'journal_corrupt') { - rebuilds.sessionIds.push(args[0].identity.sessionId) - } - return opened - } - } -}) - -const rigs: RestTestRig[] = [] - -afterEach(async () => { - folds.done.clear() - folds.after = null - rebuilds.sessionIds.length = 0 - for (const rig of rigs.splice(0)) { - await rig.dispose() - } - closeTestJournalHostDatabases() - vi.restoreAllMocks() -}) - -async function newRig(): Promise { - const rig = await createRestTestRig() - rigs.push(rig) - return rig -} - -/** Every chat's row dropped, as version 5 finds a database an older build wrote. */ -function upgradeToEmptyStatusTable(rig: RestTestRig): void { - openTestJournalHostDatabase(rig.root).db.prepare('DELETE FROM journal_session_state').run() -} - -const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds - -/** Startup up to the listing, as the host runs it; answers what the background pass is owed. */ -async function startupThroughListing(rig: RestTestRig): Promise { - const listed = listedIds(rig) - await rig.host.reconcileRestartLeases() - const background = rig.host.seedStoredStatuses(listed) - await rig.host.settleOwedSessions(listed) - return background -} - -/** A chat whose turn was still running when Orca died. */ -async function crashMidTurn(rig: RestTestRig, sessionId: string): Promise { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value - await rig.host - .collaboratorsForTests() - .sessions.get(sessionId)! - .journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } - ) -} - -/** Appends `count` short status items to each chat's history, as a stopped Orca left them. */ -function appendHistory(rig: RestTestRig, sessionIds: readonly string[], count: number): void { - const { db } = openTestJournalHostDatabase(rig.root) - db.exec('BEGIN') - for (const sessionId of sessionIds) { - const tip = liveTestJournalRows(db, sessionId).at(-1)! - for (let index = 0; index < count; index += 1) { - insertTestJournalRow(db, sessionId, { - v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION, - kind: 'item', - epoch: tip.epoch, - seq: tip.seq + 1 + index, - fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, - ts: 1_000 + index, - itemId: `${sessionId}-note-${index}`, - revision: 1, - body: { kind: 'status', text: `note ${index}` } - }) - } - } - db.exec('COMMIT') -} - -const seeded = (rig: RestTestRig, sessionId: string) => - rig.statusEvents.findIndex( - (event) => event.type === 'status' && event.session.sessionId === sessionId - ) - -describe('rowless listed chats after an upgrade', () => { - it('get their rows from their rows alone, in tab order, with no chat opened', async () => { - const rig = await newRig() - const ids = Array.from({ length: 12 }, (_, index) => `session-${index}`) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const background = await startupThroughListing(rig) - expect(background).toEqual(listedIds(rig)) - - await rig.host.restoreReadableSessions(background) - - for (const sessionId of ids) { - expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) - expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) - expect(restTestOpens(rig, sessionId)).toBe(0) - } - const order = ids.map((sessionId) => seeded(rig, sessionId)) - expect(order).toEqual(order.toSorted((a, b) => a - b)) - }) - - it('shows no status for a rowless chat until it has a row, and never seeds an unfinished one from its rows', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-fine', { message: 'done' }) - // A turn still running, and an approval still waiting, when Orca died. - const unfinished = ['session-running', 'session-prompt'] - for (const sessionId of unfinished) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value - const journal = rig.host.collaboratorsForTests().sessions.get(sessionId)!.journal - const options = { - fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, - turnScope: { kind: 'thread' as const } - } - await (sessionId === 'session-running' - ? journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - options - ) - : journal.appendItem( - { ...providerIdentity, ordinal: 50 }, - { - kind: 'approval', - title: 'Approve?', - detail: null, - options: [], - resolution: { - state: 'pending', - selectedOptionId: null, - resolvedBy: null, - resolvedAt: null - } - }, - options - )) - } - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const background = await startupThroughListing(rig) - // No row yet: no status at all, rather than a guess. - for (const sessionId of ['session-fine', ...unfinished]) { - expect(latestRestTestStatus(rig, sessionId)).toBeUndefined() - } - // The status stream's length when each chat first opened. - const openedAt = new Map() - rig.adapter.historyFilePath.mockImplementation(async (sessionId) => { - if (!openedAt.has(sessionId)) { - openedAt.set(sessionId, rig.statusEvents.length) - } - return null - }) - - await rig.host.restoreReadableSessions(background) - - expect(restTestOpens(rig, 'session-fine')).toBe(0) - expect(latestRestTestStatus(rig, 'session-fine')).toMatchObject({ status: 'idle' }) - for (const sessionId of unfinished) { - // Its first status comes from its open, which settles it: never a row derived beforehand. - expect(openedAt.get(sessionId)).toBeDefined() - expect(seeded(rig, sessionId)).toBeGreaterThanOrEqual(openedAt.get(sessionId)!) - expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) - expect(latestRestTestStatus(rig, sessionId)?.status).not.toBe('attention') - expect(latestRestTestStatus(rig, sessionId)?.status).not.toBe('working') - } - }) - - it('appends few WAL pages for a pass of 247 chats: their rows are written a slice at a time', async () => { - const rig = await newRig() - const ids = Array.from({ length: 247 }, (_, index) => `session-page-${index}`) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const background = await startupThroughListing(rig) - const { db } = openTestJournalHostDatabase(rig.root) - // An empty WAL, so its size after the pass is what the pass appended. - db.pragma('wal_checkpoint(TRUNCATE)') - const pageSize = Number(db.pragma('page_size', { simple: true })) - - await rig.host.restoreReadableSessions(background) - - const frames = - (statSync(join(rig.root, 'agent-session-journal.db-wal')).size - 32) / (pageSize + 24) - expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) - expect( - ids.map((sessionId) => restTestOpens(rig, sessionId)).every((opens) => opens === 0) - ).toBe(true) - // A commit per chat appends about 2.5 pages each, over 600 for this pass. - expect(frames).toBeLessThanOrEqual(150) - }, 120_000) - - it('accepts a send during the pass with at most one chat folded, and one slice written, ahead of it', async () => { - const rig = await newRig() - const ids = Array.from({ length: 6 }, (_, index) => `session-long-${index}`) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - const journal = rig.host.collaboratorsForTests().sessions.get(sessionId)!.journal - const fence = rig.store.getRecord(sessionId)!.lease.runtimeFence - // Long enough that its fold takes several parts, as a long chat's does. - for (let index = 0; index < 1_200; index += 1) { - await journal.appendItem( - { provider: 'orca', clientMessageId: `${sessionId}-note-${index}` }, - { kind: 'status', text: `${index} ${'x'.repeat(400)}` }, - { fence, turnScope: { kind: 'thread' } } - ) - } - } - await restTestChat(rig, 'session-send', { message: 'first' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - // The chat the send goes to keeps its row; the long ones are the pass's. - await rig.boot() - const background = (await startupThroughListing(rig)).filter((id) => id !== 'session-send') - const derivedCount = () => - ids.filter((sessionId) => readTestJournalSessionStatus(rig.root, sessionId) !== null).length - const foldedCount = () => ids.filter((sessionId) => folds.done.has(sessionId)).length - - let ticks = 0 - let ticking = true - const tick = (): void => { - if (ticking) { - ticks += 1 - setImmediate(tick) - } - } - setImmediate(tick) - - const pass = rig.host.restoreReadableSessions(background) - await new Promise((resolve) => setImmediate(resolve)) - const before = foldedCount() - const sent = await sendRestTestMessage(rig, 'session-send', 'during the pass') - const after = foldedCount() - await pass - ticking = false - - expect(sent).toMatchObject({ ok: true }) - expect(after - before).toBeLessThanOrEqual(1) - expect(after).toBeLessThan(ids.length) - expect(derivedCount()).toBe(ids.length) - // Each long chat folds over several tasks, not one. - expect(ticks).toBeGreaterThanOrEqual(2 * ids.length) - }, 120_000) - - it('gives each short chat its own task, so a send waits for at most one fold and one slice write', async () => { - const rig = await newRig() - // Each folds in a single part, so only a yield between chats splits the pass. - const ids = Array.from({ length: 60 }, (_, index) => `session-short-${index}`) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await restTestChat(rig, 'session-send', { message: 'first' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - appendHistory(rig, ids, 150) - upgradeToEmptyStatusTable(rig) - await rig.boot() - const background = (await startupThroughListing(rig)).filter((id) => id !== 'session-send') - const derivedCount = () => - ids.filter((sessionId) => readTestJournalSessionStatus(rig.root, sessionId) !== null).length - const foldedCount = () => ids.filter((sessionId) => folds.done.has(sessionId)).length - let ticks = 0 - let ticking = true - const tick = (): void => { - if (ticking) { - ticks += 1 - setImmediate(tick) - } - } - setImmediate(tick) - - const pass = rig.host.restoreReadableSessions(background) - await new Promise((resolve) => setImmediate(resolve)) - const before = foldedCount() - const sent = await sendRestTestMessage(rig, 'session-send', 'during the pass') - const after = foldedCount() - await pass - ticking = false - - expect(sent).toMatchObject({ ok: true }) - expect(after - before).toBeLessThanOrEqual(1) - expect(after).toBeLessThan(ids.length) - expect(derivedCount()).toBe(ids.length) - expect(ticks).toBeGreaterThanOrEqual(ids.length) - }, 120_000) - - it('writes the row of a chat whose tab closes mid-pass, but gives it no sidebar row', async () => { - const rig = await newRig() - for (const sessionId of ['session-kept', 'session-closing']) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const background = await startupThroughListing(rig) - // The tab closes after the chat's fold, before its slice is written. - folds.after = async (sessionId) => { - if (sessionId === 'session-closing') { - await rig.host.setSessionTabVisibility(sessionId, false) - } - } - - await rig.host.restoreReadableSessions(background) - - expect(readTestJournalSessionStatus(rig.root, 'session-closing')).toMatchObject({ - lifecycle: 'idle' - }) - expect(seeded(rig, 'session-closing')).toBe(-1) - expect(restTestOpens(rig, 'session-closing')).toBe(0) - expect(latestRestTestStatus(rig, 'session-kept')).toMatchObject({ status: 'idle' }) - }) - - it("stores an unfinished chat's row unshown, so a tab closed before its open is still settled on the next boot", async () => { - const rig = await newRig() - await crashMidTurn(rig, 'session-crashed') - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const background = await startupThroughListing(rig) - // Closed after its fold, so the pass's open of it never happens. - folds.after = async (sessionId) => { - if (sessionId === 'session-crashed') { - await rig.host.setSessionTabVisibility(sessionId, false) - } - } - - await rig.host.restoreReadableSessions(background) - - expect(restTestOpens(rig, 'session-crashed')).toBe(0) - expect(seeded(rig, 'session-crashed')).toBe(-1) - expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ - lifecycle: 'running' - }) - expect(readUnsettledJournalSessionIds(openTestJournalHostDatabase(rig.root).db)).toEqual([ - 'session-crashed' - ]) - - // The next boot selects it from its row and settles it, tab or no tab. - folds.after = null - await rig.crash() - await rig.boot() - await startupThroughListing(rig) - expect(restTestOpens(rig, 'session-crashed')).toBeGreaterThan(0) - expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ - lifecycle: 'idle' - }) - }) - - it('gives a corrupt rowless chat no row, so the next boot still opens it and its rebuild runs', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-corrupt', { message: 'asked' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - const { db } = openTestJournalHostDatabase(rig.root) - const tip = liveTestJournalRows(db, 'session-corrupt').at(-1)! - insertTestJournalRowJson(db, 'session-corrupt', tip.seq + 1, '{') - upgradeToEmptyStatusTable(rig) - expect(loadTestJournal(rig.root, 'session-corrupt')).toMatchObject({ corrupt: true }) - await rig.boot() - // Orca quits after the pass's rows, before the restore that follows opens anything. - const restore = vi - .spyOn(StructuredAgentSessionReadableRestorer.prototype, 'restore') - .mockResolvedValue(undefined) - - await rig.host.restoreReadableSessions(await startupThroughListing(rig)) - - expect(folds.done.has('session-corrupt')).toBe(true) - expect(restore).toHaveBeenCalledWith(['session-corrupt']) - expect(readTestJournalSessionStatus(rig.root, 'session-corrupt')).toBeNull() - expect(seeded(rig, 'session-corrupt')).toBe(-1) - - restore.mockRestore() - await rig.crash() - await rig.boot() - const background = await startupThroughListing(rig) - expect(background).toEqual(['session-corrupt']) - await rig.host.restoreReadableSessions(background) - - expect(restTestOpens(rig, 'session-corrupt')).toBeGreaterThan(0) - expect(rebuilds.sessionIds).toEqual(['session-corrupt']) - }) - - it('folds a chat again after its unsynced row is lost, to the same row and no wrong status', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-settled', { message: 'asked session-settled' }) - await crashMidTurn(rig, 'session-unfinished') - const ids = ['session-settled', 'session-unfinished'] - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - // Only the pass's rows: no open, so the unfinished chat keeps the row the pass stored. - const restore = vi - .spyOn(StructuredAgentSessionReadableRestorer.prototype, 'restore') - .mockResolvedValue(undefined) - await rig.host.restoreReadableSessions(await startupThroughListing(rig)) - const first = ids.map((sessionId) => readTestJournalSessionStatus(rig.root, sessionId)) - const firstStatus = latestRestTestStatus(rig, 'session-settled') - expect(first[0]).toMatchObject({ lifecycle: 'idle' }) - expect(first[1]).toMatchObject({ lifecycle: 'running' }) - - // The unsynced commit is lost: a power cut, then a boot that finds neither row. - await rig.crash() - upgradeToEmptyStatusTable(rig) - folds.done.clear() - await rig.boot() - const eventsBefore = rig.statusEvents.length - await rig.host.restoreReadableSessions(await startupThroughListing(rig)) - - expect(ids.every((sessionId) => folds.done.has(sessionId))).toBe(true) - expect(ids.map((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toEqual(first) - // Nothing published in between but the settled chat's own row, as the first boot published it. - const between = rig.statusEvents.slice(eventsBefore) - expect( - between.every( - (event) => event.type !== 'status' || event.session.sessionId === 'session-settled' - ) - ).toBe(true) - expect(latestRestTestStatus(rig, 'session-settled')).toEqual(firstStatus) - - // Each row equals what an open writes for the chat. - restore.mockRestore() - upgradeToEmptyStatusTable(rig) - for (const sessionId of ids) { - await rig.host.history({ sessionId, direction: 'tail' }) - } - expect(readTestJournalSessionStatus(rig.root, 'session-settled')).toEqual(first[0]) - // The open settles what the pass only stored: never shown as running. - expect(readTestJournalSessionStatus(rig.root, 'session-unfinished')).toMatchObject({ - lifecycle: 'idle' - }) - expect(latestRestTestStatus(rig, 'session-unfinished')?.status).not.toBe('working') - }) -}) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 60ff1930ba81..2bc29715623f 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -90,8 +90,8 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } // Startup already seeded every settled chat's row and is settling the ones a gone process left - // with work; this opens only what stored status could not answer (no row yet, a row this build - // cannot read, a per-chat file not yet copied). + // with work; this opens only what stored status could not answer (a row this build cannot read, + // a per-chat file not yet copied, a corrupt history). // Whoever answers with this list starts it, once that answer is out. const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds this.owedStructuredAgentSessionHistoryRestore = host diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index 24587bb241b4..b192e30194dc 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -21,6 +21,7 @@ type StartupHostMembers = Partial< /** A host offering the startup step's seed and settle, plus the members a test drives. */ function installStartupHost(members: StartupHostMembers): void { const host = { + catchUpMissingStatuses: async () => undefined, seedStoredStatuses: (ids: readonly string[]) => [...ids], settleOwedSessions: async () => undefined, ...members diff --git a/src/main/runtime/saved-structured-agent-session-restoration.test.ts b/src/main/runtime/saved-structured-agent-session-restoration.test.ts index d677be52eed8..8a28f0fd8d06 100644 --- a/src/main/runtime/saved-structured-agent-session-restoration.test.ts +++ b/src/main/runtime/saved-structured-agent-session-restoration.test.ts @@ -1,9 +1,7 @@ import { describe, expect, it } from 'vitest' import type { Tab } from '../../shared/tab-types' import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' -import { orderOnScreenStructuredAgentSessionsFirst } from '../../shared/saved-on-screen-structured-agent-sessions' import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' -import { runStructuredAgentSessionStartupStep } from './structured-agent-session-startup-step' function tab(input: Partial & Pick): Tab { const { id, ...overrides } = input @@ -84,79 +82,4 @@ describe('saved structured session restoration targets', () => { expect(collectSavedStructuredAgentSessionIds(saved)).toEqual(['session-codex']) }) - - it("fills the chats on screen first: the active worktree's visible tabs, then every other worktree's", async () => { - const tabs = { - 'workspace-1': [ - tab({ id: 'a-1', entityId: 'session-a-1' }), - tab({ id: 'a-2', entityId: 'session-a-2' }) - ], - 'workspace-2': [ - tab({ id: 'b-1', worktreeId: 'workspace-2', entityId: 'session-b-1' }), - tab({ - id: 'b-2', - worktreeId: 'workspace-2', - entityId: 'session-b-2', - agentSessionAgent: 'claude' - }), - tab({ - id: 'b-remote', - worktreeId: 'workspace-2', - entityId: 'session-b-remote', - executionHostId: 'ssh:build' - }) - ] - } - const group = (id: string, worktreeId: string, activeTabId: string) => ({ - id, - worktreeId, - activeTabId, - tabOrder: [activeTabId] - }) - // As the renderer saves it: each group's active tab is what is on screen, the focused group's - // first; `activeTabIdByWorktree` names a terminal whatever is shown. - const saved: WorkspaceSessionState = { - ...session([], null), - activeWorktreeId: 'workspace-2', - unifiedTabs: tabs, - tabGroups: { - 'workspace-1': [group('g-a', 'workspace-1', 'a-2')], - 'workspace-2': [ - group('g-b-left', 'workspace-2', 'b-1'), - group('g-b-right', 'workspace-2', 'b-2') - ] - }, - activeGroupIdByWorktree: { 'workspace-1': 'g-a', 'workspace-2': 'g-b-right' }, - activeTabIdByWorktree: { 'workspace-1': 'terminal-a', 'workspace-2': 'terminal-b' } - } - // The pass's order: the host's tab order, which knows nothing of what is on screen. - const listed = ['session-a-1', 'session-a-2', 'session-b-1', 'session-b-2', 'session-b-remote'] - - expect(orderOnScreenStructuredAgentSessionsFirst(listed, saved)).toEqual([ - 'session-b-2', - 'session-b-1', - 'session-a-2', - 'session-a-1', - 'session-b-remote' - ]) - // Only reorders the chats it is given. - expect(orderOnScreenStructuredAgentSessionsFirst(['session-a-1'], saved)).toEqual([ - 'session-a-1' - ]) - expect(orderOnScreenStructuredAgentSessionsFirst(listed, null)).toEqual(listed) - - const host = { - reconcileRestartLeases: async () => undefined, - getPersistedVisibleSessionTabIndex: () => ({ present: true, sessionIds: listed }), - seedStoredStatuses: (ids: readonly string[]) => [...ids], - settleOwedSessions: async () => undefined - } - expect(await runStructuredAgentSessionStartupStep(host, saved, () => undefined)).toEqual([ - 'session-b-2', - 'session-b-1', - 'session-a-2', - 'session-a-1', - 'session-b-remote' - ]) - }) }) diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index 11221f2da156..9c1ee2c3cfec 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -5,7 +5,6 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' -import { orderOnScreenStructuredAgentSessionsFirst } from '../../shared/saved-on-screen-structured-agent-sessions' import { collectSavedStructuredAgentSessionIds } from './saved-structured-agent-session-restoration' import { ensureStructuredAgentSessionHostUnlessRefused } from './structured-agent-session-host-refusal' import type { @@ -16,7 +15,7 @@ import type { /** Named from the host so a rename fails to compile. */ export type StructuredAgentSessionStartupHost = Pick< StructuredAgentSessionHost, - 'reconcileRestartLeases' | 'seedStoredStatuses' | 'settleOwedSessions' + 'reconcileRestartLeases' | 'catchUpMissingStatuses' | 'seedStoredStatuses' | 'settleOwedSessions' > & StructuredAgentSessionListingHost @@ -37,11 +36,13 @@ export async function runStructuredAgentSessionStartupStep( ): Promise { await host.reconcileRestartLeases() const listedIds = listedStructuredAgentSessionIds(host, savedSession) + // Before the listing answers: the first launch after the upgrade derives the rows listed chats + // lack, so the seed and the settle below read stored status for every one of them. + await host.catchUpMissingStatuses(listedIds) const background = host.seedStoredStatuses(listedIds) // Not awaited here: the tab list and paint never wait on it; chat commands do. onSettling(host.settleOwedSessions(listedIds)) - // The rows the window shows at launch fill first. - return orderOnScreenStructuredAgentSessionsFirst(background, savedSession) + return background } /** The chats with a tab, for the startup step and the tab restore alike: the host's persisted tab diff --git a/src/main/runtime/structured-agent-session-support-probe.test.ts b/src/main/runtime/structured-agent-session-support-probe.test.ts index 604d79c91997..7c869859fe0d 100644 --- a/src/main/runtime/structured-agent-session-support-probe.test.ts +++ b/src/main/runtime/structured-agent-session-support-probe.test.ts @@ -35,6 +35,7 @@ function stubStructuredHostInstall(runtime: OrcaRuntimeService): { // still returns the right answer and fails on the install effects alone. const host = { reconcileRestartLeases: vi.fn(async () => {}), + catchUpMissingStatuses: vi.fn(async () => undefined), seedStoredStatuses: vi.fn((ids: readonly string[]) => [...ids]), settleOwedSessions: vi.fn(async () => undefined), supportsCreate: (location: { executionHostId: string; wslDistro: string | null }) => diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index a4a1dd0fd2a1..c947d2882e91 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -157,7 +157,7 @@ function within(promise: Promise, ms = 2_000): Promise { - it('answers before any chat history is read, then gives each chat its status (T1)', async () => { + it('answers once each listed chat with no stored status has one, published, opening none (T1)', async () => { const ids = ['session-1', 'session-2', 'session-3'] for (const sessionId of ids) { await restTestChat(rig, sessionId, { message: sessionId }) @@ -165,24 +165,16 @@ describe('listing chat tabs at startup', () => { await rig.crash() await rig.boot() forgetStoredChatState() - const reads = Promise.withResolvers() - releaseHeld = reads.resolve - const restore = rig.host.restoreReadableSessions.bind(rig.host) - vi.spyOn(rig.host, 'restoreReadableSessions').mockImplementation(async (sessionIds) => { - await reads.promise - return restore(sessionIds) - }) const { listAll } = restartedRuntime() expect(await within(listAll())).toEqual(ids) - expect(ids.map((id) => readTestJournalSessionStatus(rig.root, id))).toEqual([null, null, null]) - reads.resolve() - // From the rows alone: a chat with nothing left over is not opened for its status. - await vi.waitFor(() => - ids.forEach((id) => expect(latestRestTestStatus(rig, id)).toMatchObject({ latestPrompt: id })) - ) - expect(ids.map((id) => restTestOpens(rig, id))).toEqual([0, 0, 0]) + // Derived from their rows before the answer, and published by the seed that follows. + for (const sessionId of ids) { + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) + expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ latestPrompt: sessionId }) + expect(restTestOpens(rig, sessionId)).toBe(0) + } }) it.each(['listAll', 'subscribeAll'] as const)( @@ -208,9 +200,9 @@ describe('listing chat tabs at startup', () => { const answered = method === 'listAll' ? { ids: await listAll(), atAnswer: seen() } : await subscribeAll(seen) - expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0, rows: 0 } }) - await vi.waitFor(() => expect(rows()).toBe(ids.length)) - expect(restore).toHaveBeenCalledOnce() + // Every row is written before the answer; nothing is opened before or after it. + expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0, rows: ids.length } }) + await vi.waitFor(() => expect(restore).toHaveBeenCalledOnce()) runtime.cleanupSubscriptionsForConnection('connection-1') } ) diff --git a/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts b/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts deleted file mode 100644 index e159338806b9..000000000000 --- a/src/renderer/src/store/slices/structured-chat-startup-fill-order.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -// The startup status pass fills the chats on screen first, read from the workspace session the -// renderer really saves: a chat the user is looking at is its tab group's active tab, while -// `activeTabIdByWorktree` keeps naming a terminal. - -import { beforeEach, describe, expect, it, vi } from 'vitest' -import type * as AgentStatusModule from '@/lib/agent-status' -import { buildWorkspaceSessionPayload } from '@/lib/workspace-session' -import type { Tab } from '../../../../shared/tab-types' -import { orderOnScreenStructuredAgentSessionsFirst } from '../../../../shared/saved-on-screen-structured-agent-sessions' -import { createTabsSliceMockApi } from './tabs-slice-test-harness' -import { createTestStore } from './store-test-helpers' -import { buildActiveSurfacePatch } from './tabs/tabs-surface' - -vi.mock('sonner', () => ({ toast: { info: vi.fn(), success: vi.fn(), error: vi.fn() } })) -vi.mock('@/lib/agent-status', async (importOriginal) => { - const actual = await importOriginal() - return { ...actual, detectAgentStatusFromTitle: vi.fn().mockReturnValue(null) } -}) -createTabsSliceMockApi() - -const WORKTREE = 'repo1::/tmp/feature' -const GROUP = 'group-1' - -function tab(id: string, contentType: Tab['contentType'], sortOrder: number): Tab { - return { - id, - entityId: contentType === 'agent-session' ? `session-${id.slice(-1)}` : id, - groupId: GROUP, - worktreeId: WORKTREE, - contentType, - label: id, - customLabel: null, - color: null, - sortOrder, - createdAt: sortOrder, - ...(contentType === 'agent-session' ? { agentSessionAgent: 'codex' as const } : {}) - } -} - -describe('the chats on screen at launch', () => { - let store: ReturnType - - beforeEach(() => { - store = createTestStore() - const tabs = [ - tab('terminal-1', 'terminal', 0), - tab('structured-session-1', 'agent-session', 1), - tab('structured-session-2', 'agent-session', 2), - tab('structured-session-3', 'agent-session', 3) - ] - store.setState({ - activeWorktreeId: WORKTREE, - unifiedTabsByWorktree: { [WORKTREE]: tabs }, - groupsByWorktree: { - [WORKTREE]: [ - { - id: GROUP, - worktreeId: WORKTREE, - activeTabId: 'terminal-1', - tabOrder: tabs.map(({ id }) => id) - } - ] - }, - activeGroupIdByWorktree: { [WORKTREE]: GROUP }, - tabsByWorktree: { - [WORKTREE]: [ - { - id: 'terminal-1', - ptyId: 'pty-1', - worktreeId: WORKTREE, - title: 'Terminal 1', - customTitle: null, - color: null, - sortOrder: 0, - createdAt: 0 - } - ] - }, - ptyIdsByTabId: { 'terminal-1': ['pty-1'] }, - activeTabIdByWorktree: { [WORKTREE]: 'terminal-1' } - }) - }) - - it('come first in the pass, as the saved session records the chat the user is looking at', () => { - // The user looks at the third chat; then the worktree's surface is applied, as a focus does. - store.getState().activateTab('structured-session-3') - store.setState(buildActiveSurfacePatch(store.getState(), WORKTREE)) - - const saved = buildWorkspaceSessionPayload(store.getState()) - const listed = ['session-1', 'session-2', 'session-3'] - - expect(saved.activeTabIdByWorktree?.[WORKTREE]).toBe('terminal-1') - expect(orderOnScreenStructuredAgentSessionsFirst(listed, saved)).toEqual([ - 'session-3', - 'session-1', - 'session-2' - ]) - }) -}) diff --git a/src/shared/saved-on-screen-structured-agent-sessions.ts b/src/shared/saved-on-screen-structured-agent-sessions.ts deleted file mode 100644 index 4a24d7d465ec..000000000000 --- a/src/shared/saved-on-screen-structured-agent-sessions.ts +++ /dev/null @@ -1,44 +0,0 @@ -// Which structured chats a launch shows first, from the workspace session the renderer saves. - -import { LOCAL_EXECUTION_HOST_ID } from './execution-host' -import type { WorkspaceSessionState } from './workspace-session-state-types' - -/** - * The chats a launch shows first, each tab group's active tab (the active worktree's first, and in - * each worktree its focused group first), ahead of the rest in their given order. Read from the - * groups because `activeTabIdByWorktree` only ever holds a terminal's tab. Any agent: this orders, - * it restores nothing. - */ -export function orderOnScreenStructuredAgentSessionsFirst( - sessionIds: readonly string[], - session: WorkspaceSessionState | null -): string[] { - const tabsById = new Map( - Object.values(session?.unifiedTabs ?? {}) - .flat() - .map((tab) => [tab.id, tab]) - ) - const groupsByWorktree = session?.tabGroups ?? {} - const activeWorktree = session?.activeWorktreeId ?? null - const worktrees = [ - ...(activeWorktree && groupsByWorktree[activeWorktree] ? [activeWorktree] : []), - ...Object.keys(groupsByWorktree).filter((worktreeId) => worktreeId !== activeWorktree) - ] - const listed = new Set(sessionIds) - const first = new Set() - for (const worktreeId of worktrees) { - const groups = groupsByWorktree[worktreeId] ?? [] - const focused = session?.activeGroupIdByWorktree?.[worktreeId] - for (const group of [ - ...groups.filter((candidate) => candidate.id === focused), - ...groups.filter((candidate) => candidate.id !== focused) - ]) { - const tab = group.activeTabId ? tabsById.get(group.activeTabId) : undefined - const local = !tab?.executionHostId || tab.executionHostId === LOCAL_EXECUTION_HOST_ID - if (tab?.contentType === 'agent-session' && local && listed.has(tab.entityId)) { - first.add(tab.entityId) - } - } - } - return [...first, ...sessionIds.filter((sessionId) => !first.has(sessionId))] -} diff --git a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts index 2a769b62fc62..cde878840e84 100644 --- a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts +++ b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts @@ -128,6 +128,7 @@ async function setup() { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: restore and tab close call only these host members. { reconcileRestartLeases: async () => undefined, + catchUpMissingStatuses: async () => undefined, seedStoredStatuses: (ids: readonly string[]) => [...ids], settleOwedSessions: async () => undefined, restoreReadableSessions: async () => undefined, From 60037f6acbd814d6b2c1a4e4925adbbed8a8f53c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 12:55:51 -0700 Subject: [PATCH 077/107] refactor(native-chat): the startup settle's closed-chat open drops its duplicate settle check; a quit during the last fold writes nothing --- .../structured-agent-session-startup-state.ts | 2 +- ...red-agent-session-startup-status-catch-up.test.ts | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 6b0547566c0f..1084815c8475 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -279,7 +279,7 @@ async function settleClosed( deps: StructuredAgentSessionStartupStateDeps, record: AgentSessionRecord ): Promise { - if (deps.isDisposed() || deps.hasSession(record.sessionId) || !deps.canSettle(record)) { + if (deps.isDisposed() || deps.hasSession(record.sessionId)) { return false } const opened = await openStructuredAgentSessionConversationJournal(deps.openDeps, record, { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts index 407204af22b4..d82e40583fdc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts @@ -164,15 +164,18 @@ describe('the startup status catch-up', () => { expect(stored?.every(({ status }) => status?.lifecycle === 'idle')).toBe(true) }) - it('writes nothing more after quit, and the next launch redoes what is left', async () => { - const ids = Array.from({ length: 20 }, (_, index) => `session-${index}`) + it.each([ + ['mid-way', 'session-2'], + ['during the last fold', 'session-9'] + ])('writes nothing after quit %s, and the next launch redoes it', async (_when, quitDuring) => { + // Fewer than one batch, so nothing is due a write before the end. + const ids = Array.from({ length: 10 }, (_, index) => `session-${index}`) await chats(ids, { rowless: true }) const disposed = { value: false } - // Quit lands during the third fold; every task is due a yield, which sees it. let clock = 0 vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) trace.afterFold = (sessionId) => { - if (sessionId === 'session-2') { + if (sessionId === quitDuring) { disposed.value = true } } @@ -181,7 +184,6 @@ describe('the startup status catch-up', () => { expect(await catchUpMissingStatuses(stand, ids)).toBeNull() - // Quit came before the first batch was due, so nothing was written at all. expect(transaction).not.toHaveBeenCalled() expect(ids.some((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(false) From 6168359474986158e4776fe543b15a4670d55ba7 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:04:31 -0700 Subject: [PATCH 078/107] perf(native-chat): listed chats still in per-chat files are restored before the tab listing answers The startup catch-up now also restores listed chats with no epoch in the host's database whose history is still in a per-chat file, through the restart restore's own per-chat worker (open from the file, settle, publish), one at a time with each failure costing only its chat. The restore after the listing already skips them, as open. It calls the worker directly: the restore gate and the readable restorer are once-only, and going through them would have swallowed that later restore. The settle and the catch-up share one budgeted pass over leases a crash left recovering, so no lease is recovered twice. Chats with no tab stay restored on first use. --- ...d-agent-session-rest-test-per-chat-file.ts | 46 ++++++++++ .../structured-agent-session-startup-state.ts | 14 ++- ...nt-session-startup-status-catch-up.test.ts | 10 +- ...d-agent-session-startup-status-catch-up.ts | 61 ++++++++++++- ...ured-agent-session-status-catch-up.test.ts | 91 +++++++++++++++++++ ...-agent-session-tab-listing-startup.test.ts | 38 ++++++++ 6 files changed, 247 insertions(+), 13 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts new file mode 100644 index 000000000000..fb3adf434697 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts @@ -0,0 +1,46 @@ +// Moves a rest rig chat's history out of the host's database into its own per-chat file, as a +// build before the shared database left it: the format an upgrade finds before the copy. + +import { mkdir } from 'node:fs/promises' +import { dirname } from 'node:path' +import Database from '../../sqlite/sync-database' +import { + liveTestJournalRows, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { legacyJournalDatabaseFile } from '../agent-session-journal/journal-paths' +import type { RestTestRig } from './structured-agent-session-rest-test-rig' + +export async function moveRestTestChatToPerChatFile( + rig: RestTestRig, + sessionId: string +): Promise { + const host = openTestJournalHostDatabase(rig.root) + const rows = liveTestJournalRows(host.db, sessionId) + const workspaceId = rig.store.getRecord(sessionId)!.location.workspaceId + const path = legacyJournalDatabaseFile(host.legacyDirectoryFor({ workspaceId, sessionId })) + await mkdir(dirname(path), { recursive: true }) + const file = new Database(path) + try { + file.pragma('journal_mode = WAL') + file.exec(` +CREATE TABLE journal_rows (session_id TEXT NOT NULL, epoch TEXT NOT NULL, seq INTEGER NOT NULL, + ts INTEGER NOT NULL, row_json TEXT NOT NULL, PRIMARY KEY (session_id, epoch, seq)); +CREATE TABLE journal_sessions (session_id TEXT PRIMARY KEY, epoch TEXT NOT NULL, updated_at INTEGER NOT NULL); +CREATE TABLE journal_repairs (session_id TEXT PRIMARY KEY, epoch TEXT NOT NULL, + content_from INTEGER NOT NULL, repaired_at INTEGER NOT NULL);`) + file.pragma('user_version = 2') + const insert = file.prepare( + 'INSERT INTO journal_rows (session_id, epoch, seq, ts, row_json) VALUES (?, ?, ?, ?, ?)' + ) + for (const row of rows) { + insert.run(sessionId, row.epoch, row.seq, row.ts, row.rowJson) + } + file.prepare('INSERT INTO journal_sessions VALUES (?, ?, ?)').run(sessionId, rows[0]!.epoch, 1) + } finally { + file.close() + } + for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { + host.db.prepare(`DELETE FROM ${table} WHERE session_id = ?`).run(sessionId) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 1084815c8475..818e7f211720 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -85,11 +85,14 @@ export function createStructuredAgentSessionStartupState( let caughtUp: { listedIds: readonly string[]; stored: StoredJournalSessionStatus[] } | null = null let settling: Promise | null = null let settled = false + // One budgeted pass over the leases a crash left `recovering`, for whichever needs it first. + let recovering: Promise> | null = null + const recover = () => (recovering ??= resolveRecoveringLeases(deps)) return { catchUpMissingStatuses: async (listedIds) => { catchingUp += 1 try { - const stored = await catchUpMissingStatuses(deps, listedIds) + const stored = await catchUpMissingStatuses(deps, listedIds, recover) caughtUp = stored ? { listedIds, stored } : null } finally { catchingUp -= 1 @@ -101,7 +104,7 @@ export function createStructuredAgentSessionStartupState( return seedStoredStatuses(deps, listedIds, read) }, settleOwedSessions: (listedIds) => { - settling ??= settleOwedSessions(deps, listedIds).finally(() => { + settling ??= settleOwedSessions(deps, listedIds, recover).finally(() => { settled = true }) return settling @@ -167,14 +170,15 @@ function seedStoredStatuses( async function settleOwedSessions( deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[] + listedIds: readonly string[], + recover: () => Promise> ): Promise { try { const database = deps.openDeps.journalDatabase if (database.readOnly) { return } - const outlasted = await resolveRecoveringLeases(deps) + const outlasted = await recover() const listedOrder = new Map(listedIds.map((sessionId, index) => [sessionId, index])) const listed: AgentSessionRecord[] = [] const others: AgentSessionRecord[] = [] @@ -254,7 +258,7 @@ function dropUnreachableStatus( * is stopped and its death recorded, so the settle below reads one verdict per turn. */ async function resolveRecoveringLeases( deps: StructuredAgentSessionStartupStateDeps -): Promise> { +): Promise> { const recovering = deps.openDeps.store .listRecords() .filter((record) => record.lease.handoffStage === 'recovering') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts index d82e40583fdc..a97e898785e9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts @@ -127,6 +127,8 @@ function deps( return stand as unknown as StructuredAgentSessionStartupStateDeps } +const noRecovery = async (): Promise> => new Set() + const yields = () => trace.events.filter((event) => event === 'yield').length const folds = () => trace.events.filter((event) => event.startsWith('fold:')) @@ -138,7 +140,7 @@ describe('the startup status catch-up', () => { const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') trace.events = [] - const stored = await catchUpMissingStatuses(stand, ids) + const stored = await catchUpMissingStatuses(stand, ids, noRecovery) // No fold, no yield, no transaction; the seed reads what this read. expect(folds()).toEqual([]) @@ -155,7 +157,7 @@ describe('the startup status catch-up', () => { vi.spyOn(performance, 'now').mockImplementation(() => (clock += 3)) trace.events = [] - const stored = await catchUpMissingStatuses(deps(), ids) + const stored = await catchUpMissingStatuses(deps(), ids, noRecovery) expect(folds()).toHaveLength(ids.length) expect(yields()).toBeGreaterThan(0) @@ -182,14 +184,14 @@ describe('the startup status catch-up', () => { const stand = deps(disposed) const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') - expect(await catchUpMissingStatuses(stand, ids)).toBeNull() + expect(await catchUpMissingStatuses(stand, ids, noRecovery)).toBeNull() expect(transaction).not.toHaveBeenCalled() expect(ids.some((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(false) trace.afterFold = null disposed.value = false - await catchUpMissingStatuses(deps(), ids) + await catchUpMissingStatuses(deps(), ids, noRecovery) expect(ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(true) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts index 99ffa2b47ada..93197340bef4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts @@ -1,6 +1,8 @@ // The first launch after the upgrade to stored status: listed chats with history here and no status // row get their rows from their rows alone, before the tab listing answers. The seed that follows // then publishes them, and the settle takes any a gone process left with work, as on every launch. +// Listed chats whose history is still in a per-chat file are restored now too, by the restart +// restore's own open, which settles and publishes them; the restore after the listing skips them. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import { @@ -12,6 +14,8 @@ import { readJournalSessionStatuses, type StoredJournalSessionStatus } from '../agent-session-journal/journal-session-state' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' // Rows per transaction: one commit per chat rewrites the same table and index pages each time. @@ -28,7 +32,9 @@ const TASK_BUDGET_MS = 16 */ export async function catchUpMissingStatuses( deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[] + listedIds: readonly string[], + /** The startup settle's one budgeted recovery pass; answers the leases it left unverified. */ + recover: () => Promise> ): Promise { const database = deps.openDeps.journalDatabase if (database.readOnly) { @@ -37,14 +43,16 @@ export async function catchUpMissingStatuses( try { const stored = readJournalSessionStatuses(database.db, listedIds) const missing = stored.flatMap(({ sessionId, status }) => (status ? [] : [sessionId])) - // Every listed chat has its row: no fold, no yield, no transaction. - if (missing.length === 0) { + const inFiles = listedInPerChatFiles(deps, listedIds, stored) + // Every listed chat has its row: no fold, no yield, no transaction, no open. + if (missing.length === 0 && inFiles.length === 0) { return stored } - const written = await foldAndWrite(deps, missing) + const written = missing.length > 0 ? await foldAndWrite(deps, missing) : new Map() if (!written) { return null } + await restoreFromPerChatFiles(deps, inFiles, recover) return stored.map((row) => { const status = written.get(row.sessionId) return status ? { ...row, status } : row @@ -144,3 +152,48 @@ function writeBatch( return [] } } + +/** Listed chats with no epoch here whose history is still in a per-chat file. */ +function listedInPerChatFiles( + deps: StructuredAgentSessionStartupStateDeps, + listedIds: readonly string[], + stored: readonly StoredJournalSessionStatus[] +): AgentSessionRecord[] { + const inDatabase = new Set(stored.map(({ sessionId }) => sessionId)) + return listedIds.flatMap((sessionId) => { + const record = inDatabase.has(sessionId) ? null : deps.openDeps.store.getRecord(sessionId) + return record && + deps.canSettle(record) && + !deps.hasSession(sessionId) && + hasHistoryOutsideJournalDatabase(deps.openDeps.journalDatabase, record) + ? [record] + : [] + }) +} + +/** + * Their restore, now rather than after the listing: the restart restore's per-chat worker opens + * each one from its file, settles what a gone process left and publishes its status, one at a time, + * a failed chat costing only itself. A lease whose recovery outlasted its budget is not recovered + * again beside it; its chat opens unverified, as in the settle. + */ +async function restoreFromPerChatFiles( + deps: StructuredAgentSessionStartupStateDeps, + records: AgentSessionRecord[], + recover: () => Promise> +): Promise { + if (records.length === 0 || deps.isDisposed()) { + return + } + try { + const outlasted = await recover() + await deps.restoreListed(records, (sessionId) => + outlasted.has(sessionId) ? Promise.resolve(true) : deps.resolveRecovery(sessionId) + ) + } catch (error) { + deps.openDeps.logger.warn('restoring chats still in per-chat files at startup failed', { + scope: 'startup-status-catch-up', + error + }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts index 6a5ba6e7d330..1be4a4840b05 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts @@ -7,6 +7,7 @@ import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import type * as JournalSessionStateModule from '../agent-session-journal/journal-session-state' import type * as JournalRecoveryModule from './agent-session-journal-recovery' +import type * as PerSessionImportModule from '../agent-session-journal/journal-per-session-import' import { closeTestJournalHostDatabases, insertTestJournalRowJson, @@ -24,6 +25,7 @@ import { latestRestTestStatus, restTestOpens } from './structured-agent-session-rest-test-observations' +import { moveRestTestChatToPerChatFile } from './structured-agent-session-rest-test-per-chat-file' // The listed chats' stored-status read fails once, when set: the catch-up's own read. const reads = vi.hoisted(() => { @@ -67,11 +69,39 @@ vi.mock('./agent-session-journal-recovery', async (importOriginal) => { } }) +// Each preview of a chat still in its per-chat file, and the chats whose file reads as damaged. +const previews = vi.hoisted(() => { + const seen: { sessionIds: string[]; damaged: Set } = { + sessionIds: [], + damaged: new Set() + } + return seen +}) + +vi.mock('../agent-session-journal/journal-per-session-import', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + previewPerSessionJournal: async ( + ...args: Parameters + ) => { + const { sessionId } = args[0].identity + previews.sessionIds.push(sessionId) + if (previews.damaged.has(sessionId)) { + throw new Error('file is not a database') + } + return actual.previewPerSessionJournal(...args) + } + } +}) + const rigs: RestTestRig[] = [] afterEach(async () => { reads.failNext = false rebuilds.sessionIds.length = 0 + previews.sessionIds.length = 0 + previews.damaged.clear() for (const rig of rigs.splice(0)) { await rig.dispose() } @@ -314,4 +344,65 @@ describe('listed chats with no stored status after the upgrade', () => { lifecycle: 'idle' }) }) + + it('restores a listed chat still in its per-chat file before the listing answers, read once', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-file', { message: 'asked session-file' }) + await restTestChat(rig, 'session-closed-file', { message: 'asked', listed: false }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + await moveRestTestChatToPerChatFile(rig, 'session-file') + await moveRestTestChatToPerChatFile(rig, 'session-closed-file') + await rig.boot() + + const background = await startup(rig) + + // Restored from its file by the time the listing answers: open, settled and published. + expect(rig.host.hasSession('session-file')).toBe(true) + expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ + latestPrompt: 'asked session-file' + }) + // The restore after the listing skips it: one preview, one open. + await rig.host.restoreReadableSessions(background) + expect(previews.sessionIds.filter((id) => id === 'session-file')).toEqual(['session-file']) + expect(restTestOpens(rig, 'session-file')).toBe(1) + // One with no tab waits for its first use. + expect(rig.host.hasSession('session-closed-file')).toBe(false) + expect(previews.sessionIds).not.toContain('session-closed-file') + expect(latestRestTestStatus(rig, 'session-closed-file')).toBeUndefined() + }) + + it('isolates a damaged per-chat file and a missing one: the others restore and startup goes on', async () => { + const rig = await newRig() + for (const sessionId of ['session-damaged', 'session-missing', 'session-file']) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.host.flushAllStreamedEvents() + await rig.crash() + await moveRestTestChatToPerChatFile(rig, 'session-damaged') + await moveRestTestChatToPerChatFile(rig, 'session-file') + // Its file is gone too: nothing of it anywhere but its record and tab. + for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { + openTestJournalHostDatabase(rig.root) + .db.prepare(`DELETE FROM ${table} WHERE session_id = ?`) + .run('session-missing') + } + previews.damaged.add('session-damaged') + await rig.boot() + const warn = vi.spyOn(rig.host.deps.logger, 'warn') + + const background = await startup(rig) + + expect(listedIds(rig)).toEqual(['session-damaged', 'session-missing', 'session-file']) + expect(rig.host.hasSession('session-file')).toBe(true) + expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ status: 'idle' }) + expect(warn).toHaveBeenCalledWith( + 'restoring a chat for reading failed', + expect.objectContaining({ sessionId: 'session-damaged' }) + ) + expect(rig.host.hasSession('session-damaged')).toBe(false) + // Both files stay the restore's after the listing, which skips the open one; the chat with no + // file anywhere has nothing to read, and is listed all the same. + expect(background).toEqual(['session-damaged', 'session-file']) + }) }) diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index c947d2882e91..49f1933f3c22 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -1,6 +1,7 @@ // The chat tab list a client asks for at startup answers from records and the tab table. It waits // for no chat's history; each chat's history is read after the answer. +import { rm, writeFile } from 'node:fs/promises' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../shared/protocol-version' import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' @@ -20,6 +21,8 @@ import { latestRestTestStatus, restTestOpens } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-observations' +import { moveRestTestChatToPerChatFile } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file' +import { legacyJournalDatabaseFile } from '../native-chat/agent-session-journal/journal-paths' import { AgentSessionStoreTransactions } from './agent-session-store-transactions' import { OrcaRuntimeService } from './orca-runtime' import { RpcDispatcher } from './rpc/dispatcher' @@ -177,6 +180,41 @@ describe('listing chat tabs at startup', () => { } }) + it('lists chats still in per-chat files, restored before the answer, a damaged or missing file included', async () => { + const ids = ['session-file', 'session-damaged', 'session-missing'] + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: sessionId }) + } + await rig.crash() + await moveRestTestChatToPerChatFile(rig, 'session-file') + await moveRestTestChatToPerChatFile(rig, 'session-damaged') + const damaged = legacyJournalDatabaseFile( + openTestJournalHostDatabase(rig.root).legacyDirectoryFor({ + workspaceId: rig.store.getRecord('session-damaged')!.location.workspaceId, + sessionId: 'session-damaged' + }) + ) + await writeFile(damaged, 'not a database') + await rm(`${damaged}-wal`, { force: true }) + for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { + openTestJournalHostDatabase(rig.root) + .db.prepare(`DELETE FROM ${table} WHERE session_id = ?`) + .run('session-missing') + } + await rig.boot() + vi.spyOn(rig.host.deps.logger, 'warn').mockImplementation(() => undefined) + const { listAll } = restartedRuntime() + + expect(await within(listAll())).toEqual(ids) + + // Restored from its file before the answer; the others are listed and block nothing. + expect(rig.host.hasSession('session-file')).toBe(true) + expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ + latestPrompt: 'session-file' + }) + expect(rig.host.hasSession('session-damaged')).toBe(false) + }) + it.each(['listAll', 'subscribeAll'] as const)( 'starts the history restore only after %s has answered, census and all', async (method) => { From 144610bedd760f7809380786b63c250f37233d03 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:02:18 -0700 Subject: [PATCH 079/107] fix(native-chat): startup recovers each lease once and on its own, and seeds only chats still listed and closed - The seed reads stored statuses after the catch-up and skips a chat whose tab closed meanwhile or that is open, so a tab closed during the catch-up gets no status row and an already restored per-chat-file chat is not queued again. - Lease recovery is per lease: each `recovering` lease is recovered once per host within its budget, started before the first fold. The pre-listing restore waits only on its own chats' leases, the settle on all, and the restore after the listing uses the same resolver, so no lease is checked twice. - Listed per-chat-file chats are restored in their own startup phase, four at a time. - A quit during the fold that fills a batch writes nothing more. - Drops the unread isSettling latch. --- docs/reference/agent-status-store.md | 6 +- .../structured-agent-session-host.ts | 6 +- ...tructured-agent-session-restart-restore.ts | 7 +- .../structured-agent-session-reveal.ts | 22 ++- ...ent-session-startup-lease-recovery.test.ts | 117 ++++++++++++++ ...ed-agent-session-startup-lease-recovery.ts | 72 +++++++++ ...t-session-startup-per-chat-file-restore.ts | 48 ++++++ ...ctured-agent-session-startup-state.test.ts | 7 +- .../structured-agent-session-startup-state.ts | 131 ++++++--------- ...nt-session-startup-status-catch-up.test.ts | 123 ++++++++------ ...d-agent-session-startup-status-catch-up.ts | 115 +++---------- ...ured-agent-session-status-catch-up.test.ts | 152 +++++++++++++++++- ...tore-structured-agent-session-tabs-once.ts | 7 +- ...runtime-structured-session-restore.test.ts | 1 + .../structured-agent-session-startup-step.ts | 13 +- ...ctured-agent-session-support-probe.test.ts | 1 + ...ed-session-reopened-tab-close.unit.test.ts | 1 + 17 files changed, 577 insertions(+), 252 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 5e102624e579..b29b06f0e23a 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -135,7 +135,11 @@ Two rules the ingest must keep: chat last written before that table existed has no row at first: a listed chat gets one at startup, before the tab listing answers, computed from its journal rows without opening it, so the republish and the settle below cover - it too; any other chat gets one when it is opened, which also settles it. + it too; any other chat gets one when it is opened, which also settles it. A + listed chat whose history is still in an older build's per-chat file is + opened from that file before the tab listing answers, which publishes its + status, and gets its row with its first write. A corrupt chat gets no row + until its open rebuilds it. At host startup the host republishes a settled chat's row into the store without opening the chat, and settles a chat a gone process left with work first. A structured row in `last-status.json` would diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index caef9bb85be9..bf7efbedc75e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -85,9 +85,7 @@ export class StructuredAgentSessionHost { private readonly subscribers = this.clientDelivery.subscribers private readonly tasks = new StructuredAgentSessionTaskQueue() private readonly runtimeState: StructuredAgentSessionHostRuntimeState - private readonly reconcileLeases: ( - sessionId: string - ) => Promise + private readonly reconcileLeases: ReturnType private readonly restore: ReturnType private readonly lifetime: StructuredAgentSessionConversationLifetime private readonly conversationDelivery: ReturnType< @@ -239,6 +237,8 @@ export class StructuredAgentSessionHost { restoreReadableSessions = (ids?: readonly string[]) => this.restore.restoreReadableSessions(ids) // Startup, from each chat's stored state: see `structured-agent-session-startup-state`. catchUpMissingStatuses = (ids: readonly string[]) => this.restore.catchUpMissingStatuses(ids) + restoreListedFromPerChatFiles = (ids: readonly string[]) => + this.restore.restoreListedFromPerChatFiles(ids) seedStoredStatuses = (ids: readonly string[]) => this.restore.seedStoredStatuses(ids) settleOwedSessions = (ids: readonly string[]) => this.restore.settleOwedSessions(ids) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 3c5d0ddd67ba..d261fab11047 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -86,7 +86,10 @@ async function restoreOneStructuredAgentSessionReadUnderSerialize( } export async function restoreStructuredAgentSessionsOnRestart( - input: StructuredAgentSessionReadRestoreDeps & { records: AgentSessionRecord[] } + input: StructuredAgentSessionReadRestoreDeps & { + records: AgentSessionRecord[] + concurrency?: number + } ): Promise { const [first] = input.records if (!first) { @@ -107,7 +110,7 @@ export async function restoreStructuredAgentSessionsOnRestart( } const results = await mapSettledWithConcurrency( input.records, - JOURNAL_RESTORE_CONCURRENCY, + input.concurrency ?? JOURNAL_RESTORE_CONCURRENCY, async ({ sessionId }) => { // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. await yieldToEventLoop() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index ea2e1615712e..5ec2920086a6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -70,8 +70,8 @@ export async function revealStructuredAgentSession( } } -/** The host's startup restore: reconcile, then seed and settle from the state stored beside each - * journal, then open in the background what that state cannot answer. Its lease bookkeeping is a +/** The host's startup restore: reconcile, then catch up, seed and settle from the state stored beside + * each journal, then open in the background what that state cannot answer. Its lease bookkeeping is a * reader's, which never fails a read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, @@ -108,29 +108,37 @@ export function createStructuredAgentSessionHostRestore( ), ...rest } - const restorer = new StructuredAgentSessionReadableRestorer({ ...readRestore, supportsRecord }) - const gate = new StructuredAgentSessionRestartRestoreGate() const startup = createStructuredAgentSessionStartupState({ openDeps: deps, canSettle, seedStatus, resolveRecovery: readRestore.resolveRecovery, - restoreListed: (records, resolveListedRecovery) => + restoreListed: (records, resolveListedRecovery, concurrency) => restoreStructuredAgentSessionsOnRestart({ ...readRestore, records, - resolveRecovery: resolveListedRecovery + resolveRecovery: resolveListedRecovery, + concurrency }), recoveryBudgetMs: deps.startupRecoveryBudgetMs, serialize: rest.serialize, hasSession: rest.hasSession, + isListed: readRestore.isListed, isDisposed: rest.isDisposed }) + // Startup's resolver: a lease startup recovered, or gave up on, is not recovered again. + const restorer = new StructuredAgentSessionReadableRestorer({ + ...readRestore, + resolveRecovery: startup.recoverLease, + supportsRecord + }) + const gate = new StructuredAgentSessionRestartRestoreGate() return { reconcileRestartLeases: async () => { await reconcile('startup') }, - // The listed chats the tab list left to it: still in a per-chat file, corrupt, or unreadable. + // The listed chats the tab list left to it: corrupt, unreadable, or a per-chat file whose open + // failed before the listing. restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), ...startup } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.test.ts new file mode 100644 index 000000000000..a70abb49e363 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.test.ts @@ -0,0 +1,117 @@ +// Startup's lease recovery is per lease: the tab listing waits only on the recoveries of the chats +// opened before it, and no lease is recovered twice, whichever restore asks for it. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' +import { + createRestTestRig, + restTestChat, + type RestTestRig +} from './structured-agent-session-rest-test-rig' +import { latestRestTestStatus } from './structured-agent-session-rest-test-observations' +import { moveRestTestChatToPerChatFile } from './structured-agent-session-rest-test-per-chat-file' + +const rigs: RestTestRig[] = [] + +afterEach(async () => { + for (const rig of rigs.splice(0)) { + await rig.dispose() + } + closeTestJournalHostDatabases() + vi.restoreAllMocks() +}) + +const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds + +/** + * A chat whose provider process was up when Orca died, left in a per-chat file when `inFile`. Its + * process check answers once, which leaves its lease `recovering`, then never again; answers how + * many checks hung. + */ +async function crashWithHungRecovery( + rig: RestTestRig, + sessionId: string, + { listed, inFile }: { listed: boolean; inFile: boolean } +): Promise<() => number> { + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) + // Held mid-send, before anything releases it. + const lease = rig.store.getRecord(sessionId)!.lease + await rig.host.flushAllStreamedEvents() + await rig.crash() + await writeOlderBuildLease(rig.root, sessionId, { ...lease }) + if (inFile) { + await moveRestTestChatToPerChatFile(rig, sessionId) + } + let answered = false + let hung = 0 + rig.probeOwner.mockImplementation(async (record) => { + if (record.sessionId !== sessionId) { + return { outcome: 'pid-absent' } + } + if (!answered) { + answered = true + return { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + } + hung += 1 + return new Promise(() => {}) + }) + return () => hung +} + +/** Startup up to the listing's answer, as the host's step runs it; answers the restore's list. */ +async function startupThroughListing(rig: RestTestRig): Promise { + const listed = listedIds(rig) + await rig.host.reconcileRestartLeases() + await rig.host.catchUpMissingStatuses(listed) + await rig.host.restoreListedFromPerChatFiles(listed) + return rig.host.seedStoredStatuses(listed) +} + +const within = (work: Promise, ms: number) => + Promise.race([ + work.then(() => 'done'), + new Promise((resolve) => setTimeout(() => resolve('still waiting'), ms)) + ]) + +describe('startup lease recovery, per lease', () => { + it("answers the listing without waiting on a tabless chat's hung recovery", async () => { + const rig = await createRestTestRig() + rigs.push(rig) + await restTestChat(rig, 'session-file', { message: 'asked session-file' }) + const hung = await crashWithHungRecovery(rig, 'session-tabless', { + listed: false, + inFile: false + }) + await moveRestTestChatToPerChatFile(rig, 'session-file') + vi.spyOn(rig.host.deps.logger, 'warn') + await rig.boot({ startupRecoveryBudgetMs: 10_000 }) + expect(listedIds(rig)).toEqual(['session-file']) + + // The listed per-chat-file chat is opened before the listing; its own lease is settled. + expect(await within(startupThroughListing(rig), 5_000)).toBe('done') + expect(rig.host.hasSession('session-file')).toBe(true) + expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ status: 'idle' }) + // The tabless chat's recovery is still running, started before the listing; the settle waits. + expect(hung()).toBe(1) + await rig.host.settleOwedSessions(listedIds(rig)) + expect(hung()).toBe(1) + }, 30_000) + + it('does not check a listed per-chat-file chat again after the listing once its recovery outlasted startup', async () => { + const rig = await createRestTestRig() + rigs.push(rig) + const hung = await crashWithHungRecovery(rig, 'session-file', { listed: true, inFile: true }) + vi.spyOn(rig.host.deps.logger, 'warn') + await rig.boot({ startupRecoveryBudgetMs: 50 }) + + expect(await within(startupThroughListing(rig), 5_000)).toBe('done') + // Opened unverified before the listing: the hung check is left to the next attach or send. + expect(rig.host.hasSession('session-file')).toBe(true) + expect(rig.store.getRecord('session-file')!.lease.handoffStage).toBe('recovering') + // The restore after the listing, given every listed chat, as when startup gave it no list. + expect(await within(rig.host.restoreReadableSessions(listedIds(rig)), 3_000)).toBe('done') + expect(hung()).toBe(1) + }, 30_000) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts new file mode 100644 index 000000000000..775ed60a574b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts @@ -0,0 +1,72 @@ +// Startup's recovery of the leases a crash left `recovering`: a provider process that outlived the +// crash is stopped and its death recorded, so each settle verdict reads that evidence. Each lease is +// recovered once per host, within a budget, by whichever startup phase asks first, so one phase +// waits only on the leases of the chats it opens. + +import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' +import { withTimeout } from '../../../shared/promise-timeout-fallback' +import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' + +// Record-only (a probe and at most a process stop each), so a few at once. +const RECOVERY_CONCURRENCY = 4 +// A recovery is process probes around at most one stop (its SIGTERM grace is 5.5 s). Past this a +// probe has stopped answering: the lease stays `recovering`, unverified and never called dead, for +// the next attach or send to resolve, and startup goes on. +const RECOVERY_BUDGET_MS = 15_000 + +export type StructuredAgentSessionStartupLeaseRecovery = { + /** Starts every `recovering` lease's recovery, listed or not; ends once each has ended or + * outlasted its budget. Never rejects. */ + recoverAll: () => Promise + /** A restore's resolver. A `recovering` lease answers its one recovery, or true once that + * outlasts its budget: a second beside it could hang the same way, so its chat opens unverified. + * Any other lease is resolved as `resolveRecovery` resolves it. */ + resolve: (sessionId: string) => Promise +} + +export function createStructuredAgentSessionStartupLeaseRecovery( + deps: Pick & { + budgetMs?: number + } +): StructuredAgentSessionStartupLeaseRecovery { + const budgetMs = deps.budgetMs ?? RECOVERY_BUDGET_MS + const recoveries = new Map>() + let all: Promise | null = null + const recovering = (sessionId: string) => + deps.openDeps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' + const recoverOnce = (sessionId: string): Promise => { + let recovery = recoveries.get(sessionId) + if (!recovery) { + recovery = withTimeout(deps.resolveRecovery(sessionId), budgetMs, null).then( + (resolved) => { + if (resolved !== null) { + return resolved + } + deps.openDeps.logger.warn('a chat recovery outlasted startup; left unverified', { + scope: 'startup-recovery-timeout', + sessionId + }) + return true + } + ) + recoveries.set(sessionId, recovery) + } + return recovery + } + return { + recoverAll: () => + (all ??= forEachWithConcurrency( + deps.openDeps.store + .listRecords() + .filter((record) => record.lease.handoffStage === 'recovering'), + RECOVERY_CONCURRENCY, + async ({ sessionId }) => { + await recoverOnce(sessionId) + } + )), + resolve: (sessionId) => + recoveries.has(sessionId) || recovering(sessionId) + ? recoverOnce(sessionId) + : deps.resolveRecovery(sessionId) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts new file mode 100644 index 000000000000..310659c549bf --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts @@ -0,0 +1,48 @@ +// Listed chats whose history is still in an older build's per-chat file have no stored status to +// list them from: the restart restore's own per-chat worker opens each one from its file before +// the tab listing answers, settles what a gone process left and publishes its status, and the +// restore after the listing skips it. A chat whose open fails costs only itself and stays listed. + +import { readJournalSessionStatuses } from '../agent-session-journal/journal-session-state' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' +import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' + +// On the listing's path, as main's restore before its listing is: 4 at a time. +const PRE_LISTING_RESTORE_CONCURRENCY = 4 + +/** Never rejects. `recoverLease` resolves each chat's own lease only, so the listing waits on no + * other chat's recovery. */ +export async function restoreListedFromPerChatFiles( + deps: StructuredAgentSessionStartupStateDeps, + listedIds: readonly string[], + recoverLease: (sessionId: string) => Promise +): Promise { + const database = deps.openDeps.journalDatabase + // A newer build's database: the restore after the listing reads every listed chat as before. + if (database.readOnly || deps.isDisposed()) { + return + } + try { + const inDatabase = new Set( + readJournalSessionStatuses(database.db, listedIds).map(({ sessionId }) => sessionId) + ) + const records = listedIds.flatMap((sessionId): AgentSessionRecord[] => { + const record = inDatabase.has(sessionId) ? null : deps.openDeps.store.getRecord(sessionId) + return record && + deps.canSettle(record) && + !deps.hasSession(sessionId) && + hasHistoryOutsideJournalDatabase(database, record) + ? [record] + : [] + }) + if (records.length > 0) { + await deps.restoreListed(records, recoverLease, PRE_LISTING_RESTORE_CONCURRENCY) + } + } catch (error) { + deps.openDeps.logger.warn('restoring chats still in per-chat files at startup failed', { + scope: 'startup-per-chat-file-restore', + error + }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 411b9f410f5b..90183b298c65 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -84,6 +84,7 @@ async function crashMidTurn(rig: RestTestRig, sessionId: string): Promise async function startup(rig: RestTestRig, listed: readonly string[]) { await rig.host.reconcileRestartLeases() await rig.host.catchUpMissingStatuses(listed) + await rig.host.restoreListedFromPerChatFiles(listed) const background = rig.host.seedStoredStatuses(listed) await rig.host.settleOwedSessions(listed) await rig.host.restoreReadableSessions(background) @@ -320,8 +321,9 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(opened(rig, [...listed, ...unlisted]).toSorted()).toEqual( ['session-crashed', ...unlisted, 'session-uncopied'].toSorted() ) - // Its row is derived before the seed, so only the per-chat file is left to the restore. - expect(background).toEqual(['session-uncopied']) + // Its row is derived before the seed, and the per-chat file is opened before it: nothing is + // left to the restore after the listing. + expect(background).toEqual([]) expect(latestRestTestStatus(rig, 'session-draft')).toMatchObject({ status: 'idle' }) // Settled with no user action; the listed one is open and never showed its pre-crash work. for (const sessionId of ['session-crashed', ...unlisted]) { @@ -618,6 +620,7 @@ describe('a recovery that never answers', () => { restoreListed, serialize: (_sessionId, task) => task(), hasSession: () => false, + isListed: () => true, isDisposed: () => false, recoveryBudgetMs: 20 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 818e7f211720..f746dd369c5e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -1,21 +1,22 @@ // What host startup does with each chat's stored status, before any client lists a tab. // +// Recovery: every lease a crash left `recovering` starts resolving first (a surviving provider +// process is stopped and its death recorded), so each settle verdict below reads that evidence. // Catch-up: a listed chat with history here and no stored status yet (the first launch after the -// upgrade) gets its row from its rows alone, so what follows reads stored status for it too. Seed: every settled, listed chat's status row is published from its stored status, so session -// lists have it at paint, without opening the chat. Settle: first every lease a crash left -// `recovering` is resolved (a surviving provider process is stopped and its death recorded), so -// each verdict below is taken once, from that evidence. Then every chat whose stored status shows -// work a gone process left, listed or not, is opened once, one at a time, and its open appends the -// settlement plan. A listed one goes through the restart restore's own per-chat worker and stays -// open; any other is settled and closed, never indexed or published. Chat commands wait for the -// settle (see `StructuredAgentSessionHostDeps.commandsReady`); listing, paint and status reads do -// not. A listed chat whose history is still in a per-chat file, or is corrupt, is left to the +// upgrade) gets its row from its rows alone, so what follows reads stored status for it too. A listed +// chat whose history is still in a per-chat file is then opened from it, before the listing (see +// `structured-agent-session-startup-per-chat-file-restore`). Seed: every settled chat still listed +// and not open has its status row published from its stored status, so session lists have it at +// paint, without opening the chat. Settle: once every recovery has ended, every chat whose stored +// status shows work a gone process left, listed or not, is opened once, one at a time, and its open +// appends the settlement plan. A listed one goes through the restart restore's own per-chat worker +// and stays open; any other is settled and closed, never indexed or published. Chat commands wait +// for the settle (see `StructuredAgentSessionHostDeps.commandsReady`); listing, paint and status +// reads do not. A listed chat that is corrupt, or whose per-chat file failed to open, is left to the // background restore after the listing, which is the same worker. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' -import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { StructuredAgentSessionStatusProjection } from '../../../shared/structured-agent-session-projection' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { @@ -30,15 +31,13 @@ import { type StructuredAgentSessionConversationOpenDeps } from './structured-agent-session-conversation-open' import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' +import { + createStructuredAgentSessionStartupLeaseRecovery, + type StructuredAgentSessionStartupLeaseRecovery +} from './structured-agent-session-startup-lease-recovery' +import { restoreListedFromPerChatFiles } from './structured-agent-session-startup-per-chat-file-restore' import { catchUpMissingStatuses } from './structured-agent-session-startup-status-catch-up' -// Record-only (a probe and at most a process stop each), so a few at once. -const RECOVERY_CONCURRENCY = 4 -// A recovery is process probes around at most one stop (its SIGTERM grace is 5.5 s). Past this a -// probe has stopped answering: the lease stays `recovering`, unverified and never called dead, for -// the next attach or send to resolve, and the settle goes on. -const RECOVERY_BUDGET_MS = 15_000 - export type StructuredAgentSessionStartupStateDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { store: Pick @@ -53,70 +52,64 @@ export type StructuredAgentSessionStartupStateDeps = { * attach or send). */ resolveRecovery: (sessionId: string) => Promise /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish), resolving - * recovery through `resolveRecovery`. */ + * recovery through `resolveRecovery`, at its own concurrency unless given one. */ restoreListed: ( records: AgentSessionRecord[], - resolveRecovery: (sessionId: string) => Promise + resolveRecovery: (sessionId: string) => Promise, + concurrency?: number ) => Promise /** Tests shorten it; production takes the default. */ recoveryBudgetMs?: number serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean + /** Whether the chat still has its tab: one closed during startup is not seeded. */ + isListed: (sessionId: string) => boolean isDisposed: () => boolean } export type StructuredAgentSessionStartupState = { - /** Writes the stored status of every listed chat with history here and none yet, before the - * listing answers. Never rejects. See `catchUpMissingStatuses`. */ + /** Starts every lease recovery, then writes the stored status of every listed chat with history + * here and none yet, before the listing answers. Never rejects. */ catchUpMissingStatuses: (listedIds: readonly string[]) => Promise + /** Opens listed chats still in per-chat files, before the listing answers. Never rejects. */ + restoreListedFromPerChatFiles: (listedIds: readonly string[]) => Promise /** Seeds settled listed chats; answers the listed ids the background restore still opens. */ seedStoredStatuses: (listedIds: readonly string[]) => string[] /** Settles every chat a gone process left with work, once per host. Never rejects. */ settleOwedSessions: (listedIds: readonly string[]) => Promise - /** Startup's own chat work: true from the catch-up's start until the settle has ended. */ - isSettling: () => boolean + /** Every restart restore's lease resolver, so no `recovering` lease is recovered twice. */ + recoverLease: (sessionId: string) => Promise } export function createStructuredAgentSessionStartupState( deps: StructuredAgentSessionStartupStateDeps ): StructuredAgentSessionStartupState { - let catchingUp = 0 - // What the catch-up read for a listing, which that listing's seed reads instead of reading again. - let caughtUp: { listedIds: readonly string[]; stored: StoredJournalSessionStatus[] } | null = null + const leases = createStructuredAgentSessionStartupLeaseRecovery({ + ...deps, + budgetMs: deps.recoveryBudgetMs + }) let settling: Promise | null = null - let settled = false - // One budgeted pass over the leases a crash left `recovering`, for whichever needs it first. - let recovering: Promise> | null = null - const recover = () => (recovering ??= resolveRecoveringLeases(deps)) return { catchUpMissingStatuses: async (listedIds) => { - catchingUp += 1 - try { - const stored = await catchUpMissingStatuses(deps, listedIds, recover) - caughtUp = stored ? { listedIds, stored } : null - } finally { - catchingUp -= 1 + if (!deps.openDeps.journalDatabase.readOnly) { + void leases.recoverAll() } + await catchUpMissingStatuses(deps, listedIds) }, - seedStoredStatuses: (listedIds) => { - const read = caughtUp?.listedIds === listedIds ? caughtUp.stored : null - caughtUp = null - return seedStoredStatuses(deps, listedIds, read) - }, + restoreListedFromPerChatFiles: (listedIds) => + restoreListedFromPerChatFiles(deps, listedIds, leases.resolve), + seedStoredStatuses: (listedIds) => seedStoredStatuses(deps, listedIds), settleOwedSessions: (listedIds) => { - settling ??= settleOwedSessions(deps, listedIds, recover).finally(() => { - settled = true - }) + settling ??= settleOwedSessions(deps, listedIds, leases) return settling }, - isSettling: () => catchingUp > 0 || (settling !== null && !settled) + recoverLease: leases.resolve } } function seedStoredStatuses( deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[], - read?: readonly StoredJournalSessionStatus[] | null + listedIds: readonly string[] ): string[] { const database = deps.openDeps.journalDatabase // A newer build's database: nothing is stored this build can read, and every chat reads as it does. @@ -125,7 +118,7 @@ function seedStoredStatuses( } let stored: readonly StoredJournalSessionStatus[] try { - stored = read ?? readJournalSessionStatuses(database.db, listedIds) + stored = readJournalSessionStatuses(database.db, listedIds) } catch (error) { // Fails open: every listed chat is restored in the background, as before stored status existed. deps.openDeps.logger.warn('reading stored chat status failed', { @@ -137,6 +130,10 @@ function seedStoredStatuses( const byId = new Map(stored.map((row) => [row.sessionId, row.status])) const background: string[] = [] for (const sessionId of listedIds) { + // Its tab closed during startup, or its open (a restore, read or send) publishes its own status. + if (!deps.isListed(sessionId) || deps.hasSession(sessionId)) { + continue + } const record = deps.openDeps.store.getRecord(sessionId) if (!record) { // Its record may still be owed by the records import: the restore reads records when it runs. @@ -147,7 +144,7 @@ function seedStoredStatuses( continue } if (!byId.has(sessionId)) { - // Never sent opens nothing; a per-chat file not yet copied is read in the background. + // Never sent opens nothing; a per-chat file whose open failed is tried again in the background. if (hasHistoryOutsideJournalDatabase(database, record)) { background.push(sessionId) } @@ -171,14 +168,14 @@ function seedStoredStatuses( async function settleOwedSessions( deps: StructuredAgentSessionStartupStateDeps, listedIds: readonly string[], - recover: () => Promise> + leases: StructuredAgentSessionStartupLeaseRecovery ): Promise { try { const database = deps.openDeps.journalDatabase if (database.readOnly) { return } - const outlasted = await recover() + await leases.recoverAll() const listedOrder = new Map(listedIds.map((sessionId, index) => [sessionId, index])) const listed: AgentSessionRecord[] = [] const others: AgentSessionRecord[] = [] @@ -198,11 +195,7 @@ async function settleOwedSessions( (left, right) => (listedOrder.get(left.sessionId) ?? 0) - (listedOrder.get(right.sessionId) ?? 0) ) - // A recovery that outlasted its budget is still running: a second one beside it could hang the - // same way, so its chat is opened unverified and the next attach or send resolves it. - await deps.restoreListed(listed, (sessionId) => - outlasted.has(sessionId) ? Promise.resolve(true) : deps.resolveRecovery(sessionId) - ) + await deps.restoreListed(listed, leases.resolve) // A listed chat the worker left closed (its tab closed meanwhile) is settled like any other. others.push(...listed.filter((record) => !deps.hasSession(record.sessionId))) for (const record of others) { @@ -254,30 +247,6 @@ function dropUnreachableStatus( } } -/** Every lease a crash left `recovering`, listed or not: a provider process that outlived the crash - * is stopped and its death recorded, so the settle below reads one verdict per turn. */ -async function resolveRecoveringLeases( - deps: StructuredAgentSessionStartupStateDeps -): Promise> { - const recovering = deps.openDeps.store - .listRecords() - .filter((record) => record.lease.handoffStage === 'recovering') - const budgetMs = deps.recoveryBudgetMs ?? RECOVERY_BUDGET_MS - const outlasted = new Set() - await forEachWithConcurrency(recovering, RECOVERY_CONCURRENCY, async ({ sessionId }) => { - if ( - (await withTimeout(deps.resolveRecovery(sessionId), budgetMs, null)) === null - ) { - outlasted.add(sessionId) - deps.openDeps.logger.warn('a chat recovery outlasted startup; left unverified', { - scope: 'startup-recovery-timeout', - sessionId - }) - } - }) - return outlasted -} - /** A chat nothing holds open: settled and closed, never indexed, so it gets no status row. */ async function settleClosed( deps: StructuredAgentSessionStartupStateDeps, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts index a97e898785e9..274dffde1402 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts @@ -1,6 +1,6 @@ // The catch-up's tasks and its quit: a launch where every listed chat has its row does no work at -// all, a launch that derives rows yields only when a task has run long enough, and quit stops it -// with nothing more written, for the next launch to redo. +// all, a launch that derives rows yields only when a task has run long enough, quit stops it with +// nothing more written, for the next launch to redo, and lease recovery starts before it. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -100,7 +100,14 @@ async function chats(sessionIds: readonly string[], { rowless }: { rowless: bool } function deps( - disposed: { value: boolean } = { value: false } + disposed: { value: boolean } = { value: false }, + recovering: { + records: AgentSessionRecord[] + resolveRecovery: (id: string) => Promise + } = { + records: [], + resolveRecovery: async () => true + } ): StructuredAgentSessionStartupStateDeps { const stand = { openDeps: { @@ -110,25 +117,24 @@ function deps( ...agentSessionRecordFixture(), sessionId }), - listRecords: () => [] + listRecords: () => recovering.records }, logger: { warn: vi.fn(), error: vi.fn() } }, canSettle: (candidate: AgentSessionRecord | null): candidate is AgentSessionRecord => candidate !== null, seedStatus: vi.fn(), - resolveRecovery: async () => true, + resolveRecovery: recovering.resolveRecovery, restoreListed: async () => undefined, serialize: (_sessionId: string, task: () => Promise) => task(), hasSession: () => false, + isListed: () => true, isDisposed: () => disposed.value } // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the catch-up reads only the database, records, logger and the members above. return stand as unknown as StructuredAgentSessionStartupStateDeps } -const noRecovery = async (): Promise> => new Set() - const yields = () => trace.events.filter((event) => event === 'yield').length const folds = () => trace.events.filter((event) => event.startsWith('fold:')) @@ -140,13 +146,12 @@ describe('the startup status catch-up', () => { const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') trace.events = [] - const stored = await catchUpMissingStatuses(stand, ids, noRecovery) + await catchUpMissingStatuses(stand, ids) - // No fold, no yield, no transaction; the seed reads what this read. + // No fold, no yield, no transaction. expect(folds()).toEqual([]) expect(yields()).toBe(0) expect(transaction).not.toHaveBeenCalled() - expect(stored?.map(({ sessionId }) => sessionId).toSorted()).toEqual(ids.toSorted()) }) it('yields only once a task has run long enough, not before every chat', async () => { @@ -157,60 +162,76 @@ describe('the startup status catch-up', () => { vi.spyOn(performance, 'now').mockImplementation(() => (clock += 3)) trace.events = [] - const stored = await catchUpMissingStatuses(deps(), ids, noRecovery) + await catchUpMissingStatuses(deps(), ids) expect(folds()).toHaveLength(ids.length) expect(yields()).toBeGreaterThan(0) expect(yields()).toBeLessThan(ids.length / 2) - expect(ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(true) - expect(stored?.every(({ status }) => status?.lifecycle === 'idle')).toBe(true) + expect( + ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId)?.lifecycle === 'idle') + ).toBe(true) }) it.each([ - ['mid-way', 'session-2'], - ['during the last fold', 'session-9'] - ])('writes nothing after quit %s, and the next launch redoes it', async (_when, quitDuring) => { - // Fewer than one batch, so nothing is due a write before the end. - const ids = Array.from({ length: 10 }, (_, index) => `session-${index}`) - await chats(ids, { rowless: true }) - const disposed = { value: false } - let clock = 0 - vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) - trace.afterFold = (sessionId) => { - if (sessionId === quitDuring) { - disposed.value = true + ['mid-way', 10, 3], + ['during the last fold', 10, 10], + // The 16th fold fills a batch, which is due its write at once. + ['during the fold that fills a batch', 20, 16] + ])( + 'writes nothing after quit %s, and the next launch redoes it', + async (_when, count, quitAtFold) => { + const ids = Array.from({ length: count }, (_, index) => `session-${index}`) + await chats(ids, { rowless: true }) + const disposed = { value: false } + // Every task boundary is due a yield, but the quit lands before the next one. + let clock = 0 + vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) + trace.afterFold = () => { + if (folds().length === quitAtFold) { + disposed.value = true + } } - } - const stand = deps(disposed) - const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') + const stand = deps(disposed) + const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') - expect(await catchUpMissingStatuses(stand, ids, noRecovery)).toBeNull() + await catchUpMissingStatuses(stand, ids) - expect(transaction).not.toHaveBeenCalled() - expect(ids.some((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(false) + expect(folds()).toHaveLength(quitAtFold) + expect(transaction).not.toHaveBeenCalled() + expect(ids.some((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(false) - trace.afterFold = null - disposed.value = false - await catchUpMissingStatuses(deps(), ids, noRecovery) - expect(ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(true) - }) + trace.afterFold = null + disposed.value = false + await catchUpMissingStatuses(deps(), ids) + expect(ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(true) + } + ) - it("reads as startup's own chat work from the catch-up's start until the settle ends", async () => { + it('starts every lease recovery, listed or not, before its first fold', async () => { const ids = ['session-0', 'session-1'] await chats(ids, { rowless: true }) - const state = createStructuredAgentSessionStartupState(deps()) - const during: boolean[] = [] - trace.afterFold = () => during.push(state.isSettling()) - expect(state.isSettling()).toBe(false) - - const catchingUp = state.catchUpMissingStatuses(ids) - expect(state.isSettling()).toBe(true) - await catchingUp - expect(during).toEqual([true, true]) - const settling = state.settleOwedSessions(ids) - expect(state.isSettling()).toBe(true) - await settling - - expect(state.isSettling()).toBe(false) + const crashed = ['session-0', 'session-tabless'].map((sessionId) => { + const record = { ...agentSessionRecordFixture(), sessionId } + record.lease = { ...record.lease, sessionId, handoffStage: 'recovering' } + return record + }) + const state = createStructuredAgentSessionStartupState( + deps(undefined, { + records: crashed, + resolveRecovery: async (sessionId) => { + trace.events.push(`recover:${sessionId}`) + return true + } + }) + ) + trace.events = [] + + await state.catchUpMissingStatuses(ids) + + const firstFold = trace.events.findIndex((event) => event.startsWith('fold:')) + expect(firstFold).toBeGreaterThan(-1) + expect(trace.events.slice(0, firstFold)).toEqual( + expect.arrayContaining(['recover:session-0', 'recover:session-tabless']) + ) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts index 93197340bef4..1dfc0ba13248 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts @@ -1,8 +1,6 @@ // The first launch after the upgrade to stored status: listed chats with history here and no status // row get their rows from their rows alone, before the tab listing answers. The seed that follows // then publishes them, and the settle takes any a gone process left with work, as on every launch. -// Listed chats whose history is still in a per-chat file are restored now too, by the restart -// restore's own open, which settles and publishes them; the restore after the listing skips them. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import { @@ -10,12 +8,7 @@ import { writeJournalSessionStatuses, type FoldedJournalSessionStatus } from '../agent-session-journal/journal-session-status-backfill' -import { - readJournalSessionStatuses, - type StoredJournalSessionStatus -} from '../agent-session-journal/journal-session-state' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' +import { readJournalSessionStatuses } from '../agent-session-journal/journal-session-state' import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' // Rows per transaction: one commit per chat rewrites the same table and index pages each time. @@ -24,54 +17,38 @@ const WRITE_BATCH_CHATS = 16 const TASK_BUDGET_MS = 16 /** - * Folds and writes the row of every listed chat that has history here and none yet. Answers the - * listed chats' stored statuses, the written ones included, for the seed to publish without reading - * them again, or null when it read nothing (a read-only database, quit, or a failure, which it - * reports). Rows that changed meanwhile belong to chats opened meanwhile, which the seed skips. - * Never rejects: the seed, the settle and the listing go on whatever happens here. + * Folds and writes the row of every listed chat that has history here and none yet. Never rejects: + * a failure is reported, and the seed, the settle and the listing go on whatever happens here. */ export async function catchUpMissingStatuses( deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[], - /** The startup settle's one budgeted recovery pass; answers the leases it left unverified. */ - recover: () => Promise> -): Promise { + listedIds: readonly string[] +): Promise { const database = deps.openDeps.journalDatabase if (database.readOnly) { - return null + return } try { - const stored = readJournalSessionStatuses(database.db, listedIds) - const missing = stored.flatMap(({ sessionId, status }) => (status ? [] : [sessionId])) - const inFiles = listedInPerChatFiles(deps, listedIds, stored) - // Every listed chat has its row: no fold, no yield, no transaction, no open. - if (missing.length === 0 && inFiles.length === 0) { - return stored - } - const written = missing.length > 0 ? await foldAndWrite(deps, missing) : new Map() - if (!written) { - return null + const missing = readJournalSessionStatuses(database.db, listedIds).flatMap( + ({ sessionId, status }) => (status ? [] : [sessionId]) + ) + // Every listed chat has its row: no fold, no yield, no transaction. + if (missing.length > 0) { + await foldAndWrite(deps, missing) } - await restoreFromPerChatFiles(deps, inFiles, recover) - return stored.map((row) => { - const status = written.get(row.sessionId) - return status ? { ...row, status } : row - }) } catch (error) { deps.openDeps.logger.warn('deriving missing chat statuses at startup failed', { scope: 'startup-status-catch-up', error }) - return null } } -/** The written chats' statuses by id, or null after quit, which stops a fold within one part and - * writes nothing more. */ +/** Quit stops a fold within one part and writes nothing more, for the next launch to redo. */ async function foldAndWrite( deps: StructuredAgentSessionStartupStateDeps, sessionIds: readonly string[] -): Promise | null> { +): Promise { const quit = new AbortController() let taskStart = performance.now() const yieldWhenDue = async (): Promise => { @@ -84,18 +61,18 @@ async function foldAndWrite( quit.abort() } } - const written = new Map() + const quitting = () => quit.signal.aborted || deps.isDisposed() let batch: FoldedJournalSessionStatus[] = [] const flush = (): void => { - for (const folded of writeBatch(deps, batch)) { - written.set(folded.sessionId, folded.status) + if (!quitting()) { + writeBatch(deps, batch) } batch = [] } for (const sessionId of sessionIds) { await yieldWhenDue() - if (quit.signal.aborted || deps.isDisposed()) { - return null + if (quitting()) { + return } const record = deps.openDeps.store.getRecord(sessionId) if (!record || !deps.canSettle(record) || deps.hasSession(sessionId)) { @@ -113,11 +90,7 @@ async function foldAndWrite( flush() } } - if (quit.signal.aborted || deps.isDisposed()) { - return null - } flush() - return written } async function foldOne( @@ -141,59 +114,13 @@ async function foldOne( function writeBatch( deps: StructuredAgentSessionStartupStateDeps, batch: FoldedJournalSessionStatus[] -): FoldedJournalSessionStatus[] { +): void { try { - return writeJournalSessionStatuses(deps.openDeps.journalDatabase, batch) + writeJournalSessionStatuses(deps.openDeps.journalDatabase, batch) } catch (error) { deps.openDeps.logger.warn('writing chat statuses derived from their rows failed', { scope: 'startup-status-catch-up', error }) - return [] - } -} - -/** Listed chats with no epoch here whose history is still in a per-chat file. */ -function listedInPerChatFiles( - deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[], - stored: readonly StoredJournalSessionStatus[] -): AgentSessionRecord[] { - const inDatabase = new Set(stored.map(({ sessionId }) => sessionId)) - return listedIds.flatMap((sessionId) => { - const record = inDatabase.has(sessionId) ? null : deps.openDeps.store.getRecord(sessionId) - return record && - deps.canSettle(record) && - !deps.hasSession(sessionId) && - hasHistoryOutsideJournalDatabase(deps.openDeps.journalDatabase, record) - ? [record] - : [] - }) -} - -/** - * Their restore, now rather than after the listing: the restart restore's per-chat worker opens - * each one from its file, settles what a gone process left and publishes its status, one at a time, - * a failed chat costing only itself. A lease whose recovery outlasted its budget is not recovered - * again beside it; its chat opens unverified, as in the settle. - */ -async function restoreFromPerChatFiles( - deps: StructuredAgentSessionStartupStateDeps, - records: AgentSessionRecord[], - recover: () => Promise> -): Promise { - if (records.length === 0 || deps.isDisposed()) { - return - } - try { - const outlasted = await recover() - await deps.restoreListed(records, (sessionId) => - outlasted.has(sessionId) ? Promise.resolve(true) : deps.resolveRecovery(sessionId) - ) - } catch (error) { - deps.openDeps.logger.warn('restoring chats still in per-chat files at startup failed', { - scope: 'startup-status-catch-up', - error - }) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts index 1be4a4840b05..1cc1c546836a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts @@ -6,6 +6,7 @@ import { statSync } from 'node:fs' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import type * as JournalSessionStateModule from '../agent-session-journal/journal-session-state' +import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' import type * as JournalRecoveryModule from './agent-session-journal-recovery' import type * as PerSessionImportModule from '../agent-session-journal/journal-per-session-import' import { @@ -19,8 +20,10 @@ import { import { createRestTestRig, restTestChat, + sendRestTestMessage, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' import { latestRestTestStatus, restTestOpens @@ -47,6 +50,47 @@ vi.mock('../agent-session-journal/journal-session-state', async (importOriginal) } }) +// The catch-up's work as it happens: fold parts, finished folds and batch commits. +const work = vi.hoisted(() => { + const seen: { + parts: number + folded: string[] + commits: number + /** Runs as each fold ends, before its batch is written. */ + afterFold: ((sessionId: string) => Promise) | null + } = { parts: 0, folded: [], commits: 0, afterFold: null } + return seen +}) + +vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + foldJournalSessionStatus: async ( + ...[database, sessionId, options]: Parameters + ) => { + const yieldTask = options?.yieldTask + const folded = await actual.foldJournalSessionStatus(database, sessionId, { + ...options, + yieldTask: async () => { + work.parts += 1 + await yieldTask?.() + } + }) + work.parts += 1 + work.folded.push(sessionId) + await work.afterFold?.(sessionId) + return folded + }, + writeJournalSessionStatuses: ( + ...args: Parameters + ) => { + work.commits += 1 + return actual.writeJournalSessionStatuses(...args) + } + } +}) + // Each chat an open found corrupt and sent through its rebuild. const rebuilds = vi.hoisted(() => { const opened: { sessionIds: string[] } = { sessionIds: [] } @@ -99,6 +143,10 @@ const rigs: RestTestRig[] = [] afterEach(async () => { reads.failNext = false + work.parts = 0 + work.folded.length = 0 + work.commits = 0 + work.afterFold = null rebuilds.sessionIds.length = 0 previews.sessionIds.length = 0 previews.damaged.clear() @@ -127,6 +175,7 @@ async function startup(rig: RestTestRig): Promise { const listed = listedIds(rig) await rig.host.reconcileRestartLeases() await rig.host.catchUpMissingStatuses(listed) + await rig.host.restoreListedFromPerChatFiles(listed) const background = rig.host.seedStoredStatuses(listed) await rig.host.settleOwedSessions(listed) return background @@ -362,8 +411,10 @@ describe('listed chats with no stored status after the upgrade', () => { expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ latestPrompt: 'asked session-file' }) - // The restore after the listing skips it: one preview, one open. - await rig.host.restoreReadableSessions(background) + // Not queued for the restore after the listing, which skips it even when given every listed + // chat (its list when startup gave none): one preview, one open. + expect(background).toEqual([]) + await rig.host.restoreReadableSessions(listedIds(rig)) expect(previews.sessionIds.filter((id) => id === 'session-file')).toEqual(['session-file']) expect(restTestOpens(rig, 'session-file')).toBe(1) // One with no tab waits for its first use. @@ -401,8 +452,99 @@ describe('listed chats with no stored status after the upgrade', () => { expect.objectContaining({ sessionId: 'session-damaged' }) ) expect(rig.host.hasSession('session-damaged')).toBe(false) - // Both files stay the restore's after the listing, which skips the open one; the chat with no - // file anywhere has nothing to read, and is listed all the same. - expect(background).toEqual(['session-damaged', 'session-file']) + // The damaged file is tried again after the listing; the chat with no file anywhere has nothing + // to read, and is listed all the same. + expect(background).toEqual(['session-damaged']) + }) + it('gives a chat whose tab closes during the catch-up its row, but no status row', async () => { + const rig = await newRig() + for (const sessionId of ['session-kept', 'session-closing']) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + } + await rig.host.flushAllStreamedEvents() + await rig.crash() + upgradeToEmptyStatusTable(rig) + await rig.boot() + let closedAt = -1 + // A close is not held for startup: it lands between the catch-up's folds. + work.afterFold = async (sessionId) => { + if (sessionId === 'session-kept') { + await rig.host.setSessionTabVisibility('session-closing', false) + await rig.host.close('session-closing', 'user-close') + closedAt = rig.statusEvents.length + } + } + + await startup(rig) + + expect(closedAt).toBeGreaterThan(-1) + expect(readTestJournalSessionStatus(rig.root, 'session-closing')).toMatchObject({ + lifecycle: 'idle' + }) + expect( + rig.statusEvents + .slice(closedAt) + .filter((event) => event.type === 'status' && event.session.sessionId === 'session-closing') + ).toEqual([]) + expect(restTestOpens(rig, 'session-closing')).toBe(0) + expect(latestRestTestStatus(rig, 'session-kept')).toMatchObject({ status: 'idle' }) }) + + it('lets a send through mid-catch-up once the gate ceiling passes, behind at most one fold part and one batch commit', async () => { + const rig = await newRig() + const ids = Array.from({ length: 6 }, (_, index) => `session-long-${index}`) + for (const sessionId of ids) { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) + const journal = rig.host.collaboratorsForTests().sessions.get(sessionId)!.journal + const fence = rig.store.getRecord(sessionId)!.lease.runtimeFence + // Long enough that its fold takes several parts, as a long chat's does. + for (let index = 0; index < 1_200; index += 1) { + await journal.appendItem( + { provider: 'orca', clientMessageId: `${sessionId}-note-${index}` }, + { kind: 'status', text: `${index} ${'x'.repeat(400)}` }, + { fence, turnScope: { kind: 'thread' } } + ) + } + } + await restTestChat(rig, 'session-send', { message: 'first' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + // The chat the send goes to keeps its row; the long ones are the catch-up's. + openTestJournalHostDatabase(rig.root) + .db.prepare('DELETE FROM journal_session_state WHERE session_id != ?') + .run('session-send') + const gate = new StructuredAgentSessionStartupGate(30) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + gate.hold() + await rig.boot({ commandsReady: gate.ready }) + // Every task's budget is spent at its first check, so each fold part is a task of its own. + let clock = 0 + vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) + let sent: Promise | null = null + let ahead = { parts: -1, commits: -1 } + work.afterFold = async (sessionId) => { + if (sessionId !== ids[0]) { + return + } + // The catch-up outlasts the gate's ceiling, which lets commands go. + await new Promise((resolve) => setTimeout(resolve, 60)) + expect(gate.ready()).toBeNull() + const before = { parts: work.parts, commits: work.commits } + // A send arrives as a message: its own task, which runs once the catch-up yields. + sent = new Promise((resolve) => { + setImmediate(() => { + ahead = { parts: work.parts - before.parts, commits: work.commits - before.commits } + resolve(sendRestTestMessage(rig, 'session-send', 'during the catch-up')) + }) + }) + } + + await rig.host.catchUpMissingStatuses(listedIds(rig)) + + expect(await sent).toMatchObject({ ok: true }) + expect(ahead.parts).toBeGreaterThanOrEqual(0) + expect(ahead.parts).toBeLessThanOrEqual(1) + expect(ahead.commits).toBeLessThanOrEqual(1) + expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) + }, 120_000) }) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 2bc29715623f..c001495eabe1 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -89,9 +89,10 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const session of restored) { this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } - // Startup already seeded every settled chat's row and is settling the ones a gone process left - // with work; this opens only what stored status could not answer (a row this build cannot read, - // a per-chat file not yet copied, a corrupt history). + // Startup already seeded every settled chat's row, opened listed chats still in per-chat files, + // and is settling the ones a gone process left with work; this opens only what stored status + // could not answer (a row this build cannot read, a corrupt history, a per-chat file whose open + // failed). // Whoever answers with this list starts it, once that answer is out. const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds this.owedStructuredAgentSessionHistoryRestore = host diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index b192e30194dc..fa0b6e5d86ce 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -22,6 +22,7 @@ type StartupHostMembers = Partial< function installStartupHost(members: StartupHostMembers): void { const host = { catchUpMissingStatuses: async () => undefined, + restoreListedFromPerChatFiles: async () => undefined, seedStoredStatuses: (ids: readonly string[]) => [...ids], settleOwedSessions: async () => undefined, ...members diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index 9c1ee2c3cfec..d368db207604 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -1,6 +1,7 @@ // The structured-chat step of host startup, before any client lists a tab: the restart lease check, -// then each listed chat's status seeded from its stored status, and every chat a gone process left -// with work settled. Chat commands wait for that settle; the tab list and paint do not. +// the rows listed chats lack derived from their rows, listed chats still in per-chat files opened, +// each listed chat's status seeded from its stored status, and every chat a gone process left with +// work settled. The tab list waits for all but the settle; chat commands wait for the settle too. import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' @@ -15,7 +16,11 @@ import type { /** Named from the host so a rename fails to compile. */ export type StructuredAgentSessionStartupHost = Pick< StructuredAgentSessionHost, - 'reconcileRestartLeases' | 'catchUpMissingStatuses' | 'seedStoredStatuses' | 'settleOwedSessions' + | 'reconcileRestartLeases' + | 'catchUpMissingStatuses' + | 'restoreListedFromPerChatFiles' + | 'seedStoredStatuses' + | 'settleOwedSessions' > & StructuredAgentSessionListingHost @@ -39,6 +44,8 @@ export async function runStructuredAgentSessionStartupStep( // Before the listing answers: the first launch after the upgrade derives the rows listed chats // lack, so the seed and the settle below read stored status for every one of them. await host.catchUpMissingStatuses(listedIds) + // Listed chats an older build left in per-chat files have no stored status to list them from. + await host.restoreListedFromPerChatFiles(listedIds) const background = host.seedStoredStatuses(listedIds) // Not awaited here: the tab list and paint never wait on it; chat commands do. onSettling(host.settleOwedSessions(listedIds)) diff --git a/src/main/runtime/structured-agent-session-support-probe.test.ts b/src/main/runtime/structured-agent-session-support-probe.test.ts index 7c869859fe0d..b8f8575f528d 100644 --- a/src/main/runtime/structured-agent-session-support-probe.test.ts +++ b/src/main/runtime/structured-agent-session-support-probe.test.ts @@ -36,6 +36,7 @@ function stubStructuredHostInstall(runtime: OrcaRuntimeService): { const host = { reconcileRestartLeases: vi.fn(async () => {}), catchUpMissingStatuses: vi.fn(async () => undefined), + restoreListedFromPerChatFiles: vi.fn(async () => undefined), seedStoredStatuses: vi.fn((ids: readonly string[]) => [...ids]), settleOwedSessions: vi.fn(async () => undefined), supportsCreate: (location: { executionHostId: string; wslDistro: string | null }) => diff --git a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts index cde878840e84..c82bae075ba4 100644 --- a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts +++ b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts @@ -129,6 +129,7 @@ async function setup() { { reconcileRestartLeases: async () => undefined, catchUpMissingStatuses: async () => undefined, + restoreListedFromPerChatFiles: async () => undefined, seedStoredStatuses: (ids: readonly string[]) => [...ids], settleOwedSessions: async () => undefined, restoreReadableSessions: async () => undefined, From 67708e37227f6bfed41b69cb860d15814ccb00fb Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:16:58 -0700 Subject: [PATCH 080/107] fix(native-chat): startup's settle recovers a lease the reconcile moved only after the catch-up began If the startup lease check failed and a later call completed it while the catch-up ran, the settle still waited on the list of recovering leases taken when the catch-up started. A crashed chat with no tab was then settled without its recovery: its turn read "Couldn't confirm" instead of "Interrupted", and a provider process that outlived the crash kept running until the chat was next used. The settle now lists the recovering leases again when it starts; each lease is still recovered only once. --- ...ed-agent-session-startup-lease-recovery.ts | 10 ++-- ...ctured-agent-session-startup-state.test.ts | 56 +++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts index 775ed60a574b..6120d486e02a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts @@ -15,8 +15,9 @@ const RECOVERY_CONCURRENCY = 4 const RECOVERY_BUDGET_MS = 15_000 export type StructuredAgentSessionStartupLeaseRecovery = { - /** Starts every `recovering` lease's recovery, listed or not; ends once each has ended or - * outlasted its budget. Never rejects. */ + /** Starts the recovery of every lease `recovering` at this call, listed or not, and ends once + * each has ended or outlasted its budget; a lease a later reconcile moved there is included the + * next call. Never rejects. */ recoverAll: () => Promise /** A restore's resolver. A `recovering` lease answers its one recovery, or true once that * outlasts its budget: a second beside it could hang the same way, so its chat opens unverified. @@ -31,7 +32,6 @@ export function createStructuredAgentSessionStartupLeaseRecovery( ): StructuredAgentSessionStartupLeaseRecovery { const budgetMs = deps.budgetMs ?? RECOVERY_BUDGET_MS const recoveries = new Map>() - let all: Promise | null = null const recovering = (sessionId: string) => deps.openDeps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' const recoverOnce = (sessionId: string): Promise => { @@ -55,7 +55,7 @@ export function createStructuredAgentSessionStartupLeaseRecovery( } return { recoverAll: () => - (all ??= forEachWithConcurrency( + forEachWithConcurrency( deps.openDeps.store .listRecords() .filter((record) => record.lease.handoffStage === 'recovering'), @@ -63,7 +63,7 @@ export function createStructuredAgentSessionStartupLeaseRecovery( async ({ sessionId }) => { await recoverOnce(sessionId) } - )), + ), resolve: (sessionId) => recoveries.has(sessionId) || recovering(sessionId) ? recoverOnce(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 90183b298c65..0ecb44fcddc3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -412,6 +412,62 @@ describe('a provider process that outlived the crash (R1T-2)', () => { await startup(rig, listedIds(rig)) expect(stopOwnerProcess).not.toHaveBeenCalled() }) + + it('is stopped for an unlisted chat whose lease the reconcile moves only after the catch-up began', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-listed', { message: 'done' }) + await restTestChat(rig, 'session-closed', { message: 'asked', listed: false }) + const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value + await rig.host + .collaboratorsForTests() + .sessions.get('session-closed')! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { + fence: rig.store.getRecord('session-closed')!.lease.runtimeFence, + turnScope: { kind: 'thread' } + } + ) + const lease = rig.store.getRecord('session-closed')!.lease + await rig.host.flushAllStreamedEvents() + await rig.crash() + await writeOlderBuildLease(rig.root, 'session-closed', { ...lease }) + let alive = true + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + rig.probeOwner.mockImplementation(async (record) => + alive && record.sessionId === 'session-closed' + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } + ) + await rig.boot({ stopOwnerProcess }) + const listed = listedIds(rig) + vi.spyOn(rig.store, 'reconcileOnRestart').mockRejectedValueOnce(new Error('disk I/O error')) + + await rig.host.reconcileRestartLeases() + expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) + // A command let through mid-catch-up completes the reconcile, moving the lease to `recovering`. + const catchingUp = rig.host.catchUpMissingStatuses(listed) + await rig.host.reconcileRestartLeases() + expect(rig.store.getRecord('session-closed')!.lease.handoffStage).toBe('recovering') + await catchingUp + await rig.host.restoreListedFromPerChatFiles(listed) + rig.host.seedStoredStatuses(listed) + await rig.host.settleOwedSessions(listed) + + expect(stopOwnerProcess).toHaveBeenCalled() + expect(rig.store.getRecord('session-closed')!.lease).toMatchObject({ + deathEvidence: { kind: 'pid-absent' } + }) + // Settled from the death evidence: the interrupted turn, not an unconfirmed one. + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle', + summary: { turnOutcome: 'interruption' } + }) + expect(rig.host.hasSession('session-closed')).toBe(false) + }) }) describe('one awaited settle covers a chat whose tab closes meanwhile (R1T-4)', () => { From 334e50d1f920aa901e0d9eedeba4329ee9701d30 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:18:15 -0700 Subject: [PATCH 081/107] test(native-chat): a send let through mid-catch-up is answered before the catch-up's last fold and commit The test bounded when the send started, but read its answer only after the catch-up returned, so a send whose later steps waited for the catch-up's end still passed. It now records the catch-up's progress when the send is answered. --- ...structured-agent-session-status-catch-up.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts index 1cc1c546836a..a578303a5a63 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts @@ -522,6 +522,8 @@ describe('listed chats with no stored status after the upgrade', () => { vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) let sent: Promise | null = null let ahead = { parts: -1, commits: -1 } + // The catch-up's progress when the send was answered. + let answeredAt = { folded: -1, commits: -1 } work.afterFold = async (sessionId) => { if (sessionId !== ids[0]) { return @@ -534,7 +536,12 @@ describe('listed chats with no stored status after the upgrade', () => { sent = new Promise((resolve) => { setImmediate(() => { ahead = { parts: work.parts - before.parts, commits: work.commits - before.commits } - resolve(sendRestTestMessage(rig, 'session-send', 'during the catch-up')) + resolve( + sendRestTestMessage(rig, 'session-send', 'during the catch-up').then((answer) => { + answeredAt = { folded: work.folded.length, commits: work.commits } + return answer + }) + ) }) }) } @@ -545,6 +552,10 @@ describe('listed chats with no stored status after the upgrade', () => { expect(ahead.parts).toBeGreaterThanOrEqual(0) expect(ahead.parts).toBeLessThanOrEqual(1) expect(ahead.commits).toBeLessThanOrEqual(1) + // Answered while the catch-up still ran: before its last fold and its last commit. + expect(answeredAt.folded).toBeGreaterThan(0) + expect(answeredAt.folded).toBeLessThan(ids.length) + expect(answeredAt.commits).toBeLessThan(work.commits) expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) }, 120_000) }) From 59fd62e5732c4f57d76059becdac8e3d329cbb89 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:22:10 -0700 Subject: [PATCH 082/107] refactor(native-chat): startup status comments say what each step does now; the status fold drops its leftover batch shape - The status doc says the startup republish covers listed chats not already open, and that the settle runs after it. - Comments: the catch-up's no-row branch, a skipped chat left to its open, the restore's one-lane default against the per-chat-file pass's four, the per-chat-file open in the startup restore, the seed's listed-only scope, and the recoveries (not the lease check) recording death evidence. - The status fold splits one page into parts directly, under a name of its own; its epoch check is no longer exported, and the batch write no longer returns the chats it wrote, which nothing read. --- docs/reference/agent-status-store.md | 7 ++- .../journal-session-status-backfill.test.ts | 18 +++--- .../journal-session-status-backfill.ts | 60 +++++++++---------- ...tructured-agent-session-restart-restore.ts | 5 +- .../structured-agent-session-reveal.ts | 5 +- ...t-session-startup-per-chat-file-restore.ts | 2 +- ...d-agent-session-startup-status-catch-up.ts | 4 +- ...tore-structured-agent-session-tabs-once.ts | 8 +-- .../structured-agent-session-startup-step.ts | 5 +- 9 files changed, 59 insertions(+), 55 deletions(-) diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 64c8beef7229..754ee3947b86 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -140,9 +140,10 @@ Two rules the ingest must keep: opened from that file before the tab listing answers, which publishes its status, and gets its row with its first write. A corrupt chat gets no row until its open rebuilds it. - At host startup the host republishes a settled chat's row into the store - without opening the chat, and settles a chat a gone process left with work - first. A structured row in `last-status.json` would + At host startup, before the tab listing answers, the host republishes the + row of each settled chat that is still listed and not already open, without + opening it; after that it settles every chat a gone process left with work. + A structured row in `last-status.json` would hydrate as `restoredUnconfirmed` and fight that republish, so the serializer still skips rows carrying `structuredHost`, and hydrate still drops any found on disk. Applying one therefore also skips diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index eea360a4c2be..0711c99e70fe 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -61,13 +61,18 @@ const database = () => openTestJournalHostDatabase(root) const dropRow = (sessionId: string) => database().db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) -/** The chat's fold, and its row written as a batch of one; null when either declines. */ +/** The chat's fold, and its row written as a batch of one; null when the fold declines or the + * write leaves the chat rowless. */ async function foldAndWrite( sessionId: string, options?: Parameters[2] ): Promise { const folded = await foldJournalSessionStatus(database(), sessionId, options) - return folded && writeJournalSessionStatuses(database(), [folded]).length > 0 ? folded : null + if (!folded) { + return null + } + writeJournalSessionStatuses(database(), [folded]) + return readTestJournalSessionStatus(root, sessionId) ? folded : null } beforeEach(async () => { @@ -240,10 +245,9 @@ describe('a row from the rows alone', () => { } const transaction = vi.spyOn(database(), 'transaction') - const written = writeJournalSessionStatuses(database(), folded) + writeJournalSessionStatuses(database(), folded) expect(transaction).toHaveBeenCalledOnce() - expect(written.map(({ sessionId }) => sessionId)).toEqual([...JOURNAL_SESSION_STATE_CASES]) for (const name of JOURNAL_SESSION_STATE_CASES) { expect(readTestJournalSessionStatus(root, name)).toEqual(byOpen.get(name)) } @@ -276,7 +280,8 @@ describe('a row from the rows alone', () => { // NORMAL (1) for the batch's commit, FULL (2) for every other commit on the connection. expect(level()).toBe(2) - expect(writeJournalSessionStatuses(database(), [folded[0]!])).toHaveLength(1) + writeJournalSessionStatuses(database(), [folded[0]!]) + expect(readTestJournalSessionStatus(root, 'first')).toEqual(folded[0]!.status) expect(atCommit).toEqual([1]) expect(level()).toBe(2) @@ -357,9 +362,8 @@ describe('a row from the rows alone', () => { const rowedBefore = readTestJournalSessionStatus(root, 'rowed') publishTestJournalEpoch(database().db, 'replaced', 'epoch-replaced-later') - const written = writeJournalSessionStatuses(database(), folded) + writeJournalSessionStatuses(database(), folded) - expect(written.map(({ sessionId }) => sessionId)).toEqual(['steady']) expect(readTestJournalSessionStatus(root, 'steady')).toEqual(folded[0]!.status) expect(readTestJournalSessionStatus(root, 'appended')).toBeNull() expect(readTestJournalSessionStatus(root, 'rowed')).toEqual(rowedBefore) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index 9c49f47fa894..3e6e0bf13b61 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -19,30 +19,27 @@ import { type JournalSessionStatus } from './journal-session-state' -/** Row JSON per part, in UTF-16 units: a page of large rows is split, so each main-thread task - * handles about the same bytes (a quarter of a full page of the seed's 2.3 KB rows). */ -const IMPORT_BATCH_CHARS = 256 * 1024 +/** Row JSON per fold part, in UTF-16 units: a page of large rows is split, so each main-thread task + * handles about the same bytes. */ +const FOLD_PART_CHARS = 256 * 1024 -/** Each batch split so no part holds more than `maxChars` of row JSON; a larger row is a part alone. - * Only a batch's last part keeps its `last`. */ -function* charBoundedBatches( - batches: Iterable<{ rows: Row[]; last: boolean }>, - maxChars = IMPORT_BATCH_CHARS +/** The rows split so no part holds more than `maxChars` of row JSON; a larger row is a part alone. */ +function* charBoundedParts( + rows: readonly Row[], + maxChars: number ): Generator<{ rows: Row[]; last: boolean }> { - for (const batch of batches) { - let part: Row[] = [] - let chars = 0 - for (const row of batch.rows) { - if (part.length > 0 && chars + row.rowJson.length > maxChars) { - yield { rows: part, last: false } - part = [] - chars = 0 - } - part.push(row) - chars += row.rowJson.length + let part: Row[] = [] + let chars = 0 + for (const row of rows) { + if (part.length > 0 && chars + row.rowJson.length > maxChars) { + yield { rows: part, last: false } + part = [] + chars = 0 } - yield { rows: part, last: batch.last } + part.push(row) + chars += row.rowJson.length } + yield { rows: part, last: true } } /** A chat's status folded from its rows, and where the fold read them. */ @@ -65,7 +62,7 @@ type FoldOptions = { /** The epoch a fold of the chat would read, or null when it has nothing to fold: the database is a * newer build's, the chat has no epoch here (it is still in a per-chat file), or it has a row. */ -export function foldableJournalSessionEpoch( +function foldableJournalSessionEpoch( database: JournalHostDatabase, sessionId: string ): string | null { @@ -85,7 +82,7 @@ export async function foldJournalSessionStatus( sessionId: string, { batchRows = IMPORT_BATCH_ROWS, - batchChars = IMPORT_BATCH_CHARS, + batchChars = FOLD_PART_CHARS, yieldTask = () => yieldToEventLoop(), signal }: FoldOptions = {} @@ -107,7 +104,7 @@ export async function foldJournalSessionStatus( const rows = readJournalRowsAfter(database.db, sessionId, epoch, afterSeq, batchRows) const last = rows.at(-1) let folding = true - for (const part of charBoundedBatches([{ rows, last: true }], batchChars)) { + for (const part of charBoundedParts(rows, batchChars)) { folding = part.rows.every(fold.add) // A part per task: a long chat's whole replay in one task holds up every other chat. if (!folding || part.last) { @@ -137,26 +134,25 @@ export async function foldJournalSessionStatus( } /** Every folded chat's row in ONE transaction, each only while it still has no row and its rows are - * where its fold read them. Answers the chats written; a chat skipped is left to its open. */ + * where its fold read them; a chat skipped is left to its open. */ export function writeJournalSessionStatuses( database: JournalHostDatabase, folded: readonly FoldedJournalSessionStatus[] -): FoldedJournalSessionStatus[] { +): void { if (folded.length === 0) { - return [] + return } // Unsynced: a lost row is re-derived from the journal, and any later synced commit covers it. - return database.unsyncedTransaction((db) => - folded.filter(({ sessionId, epoch, tip, status }) => { + database.unsyncedTransaction((db) => { + for (const { sessionId, epoch, tip, status } of folded) { if ( hasJournalSessionStatus(db, sessionId) || readJournalSessionEpoch(db, sessionId) !== epoch || readJournalTip(db, sessionId, epoch) !== tip ) { - return false + continue } writeJournalSessionStatus(db, sessionId, status) - return true - }) - ) + } + }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index d261fab11047..66001d577a07 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -21,8 +21,9 @@ import type { } from './structured-agent-session-conversation-open' import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' -// One chat at a time: the restore is CPU-bound on the main thread, so more lanes only lengthen each -// event-loop turn that a user's read or send waits behind. +// One chat at a time after the listing: the restore is CPU-bound on the main thread, so more lanes +// only lengthen each event-loop turn that a user's read or send waits behind. The pass before the +// listing sets its own. const JOURNAL_RESTORE_CONCURRENCY = 1 export type StructuredAgentSessionReadRestoreDeps = { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 5ec2920086a6..53a0656b58b6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -70,8 +70,9 @@ export async function revealStructuredAgentSession( } } -/** The host's startup restore: reconcile, then catch up, seed and settle from the state stored beside - * each journal, then open in the background what that state cannot answer. Its lease bookkeeping is a +/** The host's startup restore: reconcile, catch up, open listed chats still in per-chat files, then + * seed and settle from the state stored beside each journal, then open in the background what that + * state cannot answer. Its lease bookkeeping is a * reader's, which never fails a read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts index 310659c549bf..e7cffca64f6c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts @@ -8,7 +8,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' -// On the listing's path, as main's restore before its listing is: 4 at a time. +// The listing waits for the whole pass, so a few at once: one slow file or recovery holds one lane. const PRE_LISTING_RESTORE_CONCURRENCY = 4 /** Never rejects. `recoverLease` resolves each chat's own lease only, so the listing waits on no diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts index 1dfc0ba13248..ef3513990a7e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts @@ -32,7 +32,7 @@ export async function catchUpMissingStatuses( const missing = readJournalSessionStatuses(database.db, listedIds).flatMap( ({ sessionId, status }) => (status ? [] : [sessionId]) ) - // Every listed chat has its row: no fold, no yield, no transaction. + // Only rowless chats are folded; when every listed chat has its row there is no transaction. if (missing.length > 0) { await foldAndWrite(deps, missing) } @@ -110,7 +110,7 @@ async function foldOne( } } -/** One transaction; a chat that moved or got a row since its fold is skipped and left rowless. */ +/** One transaction; a chat that moved or got a row since its fold is skipped and left to its open. */ function writeBatch( deps: StructuredAgentSessionStartupStateDeps, batch: FoldedJournalSessionStatus[] diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index c001495eabe1..2198d5d0b32c 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -89,10 +89,10 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const session of restored) { this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } - // Startup already seeded every settled chat's row, opened listed chats still in per-chat files, - // and is settling the ones a gone process left with work; this opens only what stored status - // could not answer (a row this build cannot read, a corrupt history, a per-chat file whose open - // failed). + // Startup already seeded each settled listed chat not yet open, opened listed chats still in + // per-chat files, and is settling the ones a gone process left with work; this opens only what + // stored status could not answer (a row this build cannot read, a corrupt history, a per-chat + // file whose open failed). // Whoever answers with this list starts it, once that answer is out. const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds this.owedStructuredAgentSessionHistoryRestore = host diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index d368db207604..f0d6176f75cb 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -30,8 +30,9 @@ type StructuredAgentSessionListingHost = Partial< /** * Answers the listed chats stored status could not answer, which the post-listing restore opens. - * The lease check runs first: the death evidence it writes is what the settle's verdicts read. It - * reports its own failures and never throws. + * The lease check runs first: it marks crashed leases `recovering`, and the recoveries the catch-up + * starts record the death evidence the settle's verdicts read. It reports its own failures and + * never throws. */ export async function runStructuredAgentSessionStartupStep( host: StructuredAgentSessionStartupHost, From d1bf8ae28e268a59b406a1fd61c0a0fa2ce52368 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:35:11 -0700 Subject: [PATCH 083/107] fix(native-chat): the runtime-id module stays within its 400-line limit after the main merge The merge brought main's new idle-evidence lines into a module this branch had grown for its snapshot replacements parameter, which took it past the runtime module size limit. The parameter's type is now inferred from its default and its note joins the method's doc line. --- src/main/runtime/orca-runtime-runtime-id.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/runtime/orca-runtime-runtime-id.ts b/src/main/runtime/orca-runtime-runtime-id.ts index 67e28ac4b493..c96ae4ff362c 100644 --- a/src/main/runtime/orca-runtime-runtime-id.ts +++ b/src/main/runtime/orca-runtime-runtime-id.ts @@ -3,7 +3,6 @@ import { randomUUID } from 'node:crypto' import { preserveTerminalRetirementProofs } from './mobile-session-terminal-retirement-proof' import { getStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { replaceConversationInSnapshot } from './structured-conversation-tab-replacement' -import type { ConversationReplacement } from '../native-chat/agent-session-wire/structured-conversation-command' import type { TuiAgent } from '../../shared/tui-agent' import type { RuntimeStore } from './runtime-store-contract' import type { RuntimeClientSettingsController } from './runtime-client-settings' @@ -101,13 +100,11 @@ export class OrcaRuntimeWithRuntimeId { protected mobileSessionTabsByWorktree = new Map() - /** Single host writer for mobile session snapshots; versions are total-order stamps. */ + /** Single host writer for mobile snapshots; total-order versions; a loop passes `replacements`. */ protected storeMobileSessionSnapshot( worktreeId: string, snapshot: RuntimeMobileSessionTabsSnapshot, - // A caller storing many snapshots in one synchronous loop derives these once for it. - replacements: readonly ConversationReplacement[] = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? - [] + replacements = getStructuredAgentSessionHost()?.conversationReplacements?.() ?? [] ): RuntimeMobileSessionTabsSnapshot { for (const replacement of replacements) { snapshot = replaceConversationInSnapshot(snapshot, replacement) From d46a9362872fa08fae522176bf0b192c88b0cbe8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:35:54 -0700 Subject: [PATCH 084/107] test(native-chat): the older build in the newer-row-kind downgrade test opens this database read-only and keeps the row Main's test of a newer build's row kind downgraded to an older main build that could write this database, which then deleted the row. This branch's database is at the next schema version, which that older build opens read-only, so it keeps the row and writes nothing. The test now asserts that, under the older-schema baseline this file already names. --- ...t-session-stop-event-downgrade.unit.test.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts index cc71c95082e5..dfa96246aa11 100644 --- a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts +++ b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts @@ -231,11 +231,11 @@ test("an older schema's build reads this build's journal read-only, writes nothi }, 120_000) // Why a Stop's event cannot have a row kind of its own yet: this build keeps a kind it does not know -// and goes read-only, but a build from before that deletes the journal from it. So a Stop kind ships -// its reader first and is written once no supported build lacks that reader, or is written at a -// bumped `v`. Move the baseline to the first release with this rule, and the older build keeps the -// row too. -test("this build keeps a newer build's row kind and goes read-only; a build before it deletes it", async () => { +// and goes read-only, but a build from before that deletes the journal from it when it can write it. +// So a Stop kind ships its reader first and is written once no supported build lacks that reader, or +// is written at a bumped `v`. The pinned older build is at the schema before 5, which opens this +// build's database read-only, so it keeps the row too; a released build at version 5 has this rule. +test("this build keeps a newer build's row kind and goes read-only; an older schema's build keeps it too", async () => { const directory = mkdtempSync(join(tmpdir(), 'orca-newer-kind-downgrade-')) const journals = createTrackedJournalOpener() const newerKinds = () => storedRows(directory).filter((row) => row.includes('"future-mark"')) @@ -273,18 +273,20 @@ test("this build keeps a newer build's row kind and goes read-only; a build befo expect(storedRows(directory)).toEqual(rowsBefore) expect(newerKinds()).toEqual([newer]) - const checkout = await materializeReleaseCheckout(WRITABLE_BASELINE_REF) + const checkout = await materializeReleaseCheckout(OLDER_SCHEMA_BASELINE_REF) const support = await importReleaseCheckoutModule( checkout, `${JOURNAL}/journal-host-database-test-support.ts` ) const older = releaseExport<() => OlderOpener>(support, 'createTrackedJournalOpener')() try { - await older.open({ identity: IDENTITY, stateDirectory: directory }) + const downgraded = await older.open({ identity: IDENTITY, stateDirectory: directory }) + expect(downgraded.isReadOnly).toBe(true) } finally { await older.closeAll() } - expect(newerKinds()).toEqual([]) + expect(newerKinds()).toEqual([newer]) + expect(storedRows(directory)).toEqual(rowsBefore) } finally { await journals.closeAll() rmSync(directory, { recursive: true, force: true }) From 1b94e2304e245e81b000623c13023485877981d2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:36:48 -0700 Subject: [PATCH 085/107] refactor(native-chat): the journal store stays within its length limit after the main merge Main's in-order read and resolved-item append took the store past its line limit. The status projection read is now a field bound in the constructor, as the history read already is, and the in-order read is a one-line arrow. --- .../agent-session-journal/journal-store.ts | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index a6991952bc00..75b89c7073fa 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -55,10 +55,7 @@ import type { ResolveDispatchInput } from './journal-store-contracts' import { queuedMessageConsumeHook, type JournalQueuedMessages } from './journal-queued-messages' -import type { - JournalStatusProjection, - JournalStatusProjectionState -} from './journal-status-projection' +import type { JournalStatusProjectionState } from './journal-status-projection' import { journalQueueResumeRowBuilder, journalStopEventRowBuilder @@ -99,7 +96,8 @@ export class AgentSessionJournal { * fails the open that calls it. */ readonly backfillSessionStatus: () => void readonly readSince: (cursor: AgentJournalCursor, limit?: number) => JournalReadSince - private readonly statusProjection: JournalStatusProjection + /** The status projection at this tip, projected once per commit for every reader. */ + readonly statusState: (fence: number | undefined) => JournalStatusProjectionState /** Draft rows queued while the agent works; never reducer input or owed work. */ readonly queuedMessages: JournalQueuedMessages readonly stopMarks: JournalStopMarks @@ -155,7 +153,7 @@ export class AgentSessionJournal { this.restore = collaborators.restore this.backfillSessionStatus = collaborators.backfillSessionStatus this.readSince = collaborators.readSince - this.statusProjection = collaborators.statusProjection + this.statusState = (fence) => collaborators.statusProjection.at(fence) } private get state(): JournalReducerState { @@ -233,10 +231,6 @@ export class AgentSessionJournal { snapshot = (): AgentJournalSnapshot => renderJournalState(this.state) - /** The status projection at this tip, projected once per commit for every reader. */ - statusState = (fence: number | undefined): JournalStatusProjectionState => - this.statusProjection.at(fence) - /** Visits reduced items without allocating and sorting a full snapshot. */ visitItems = ( visit: (itemId: string, sequence: number, body: AgentJournalItemBody) => void @@ -304,9 +298,7 @@ export class AgentSessionJournal { /** Reads the fold with every write issued before this call committed, and none issued after: at * once unless writes still wait behind an owed import or a running write. */ - readInOrder(read: () => T): Promise { - return this.queue.readInOrder(read) - } + readInOrder = (read: () => T): Promise => this.queue.readInOrder(read) /** Upsert by stable identity. The revision is assigned here so a caller * cannot accidentally publish a revision the reducer will drop. */ From 88fff1335e0a2559a384d4b5b05c29acce35ceaa Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:37:50 -0700 Subject: [PATCH 086/107] fix(native-chat): startup settles each closed chat after recovering its lease, from its current record If the startup lease check failed, the settle's own restore of listed chats could be the call that completed it, after the settle had already waited on the leases it then knew to be recovering. A crashed chat with no tab was then settled without its recovery and from the record read before: its provider process was never stopped and its turn read "Couldn't confirm" instead of "Interrupted". Each closed chat now has its lease recovered (once, as everywhere at startup) just before its settle, which reads the record as that recovery left it. The earlier test of this path flushed the chat's writes before the crash, so the chat owed nothing and its verdict was never re-derived; it now crashes with the turn running and checks that the chat is owed before the settle. --- ...ctured-agent-session-startup-state.test.ts | 114 ++++++++++++------ .../structured-agent-session-startup-state.ts | 7 +- 2 files changed, 83 insertions(+), 38 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 0ecb44fcddc3..1452aa9c7f61 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -416,33 +416,7 @@ describe('a provider process that outlived the crash (R1T-2)', () => { it('is stopped for an unlisted chat whose lease the reconcile moves only after the catch-up began', async () => { const rig = await newRig() await restTestChat(rig, 'session-listed', { message: 'done' }) - await restTestChat(rig, 'session-closed', { message: 'asked', listed: false }) - const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value - await rig.host - .collaboratorsForTests() - .sessions.get('session-closed')! - .journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { - fence: rig.store.getRecord('session-closed')!.lease.runtimeFence, - turnScope: { kind: 'thread' } - } - ) - const lease = rig.store.getRecord('session-closed')!.lease - await rig.host.flushAllStreamedEvents() - await rig.crash() - await writeOlderBuildLease(rig.root, 'session-closed', { ...lease }) - let alive = true - const stopOwnerProcess = vi.fn(() => { - alive = false - }) - rig.probeOwner.mockImplementation(async (record) => - alive && record.sessionId === 'session-closed' - ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } - : { outcome: 'pid-absent' } - ) - await rig.boot({ stopOwnerProcess }) + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-closed']) const listed = listedIds(rig) vi.spyOn(rig.store, 'reconcileOnRestart').mockRejectedValueOnce(new Error('disk I/O error')) @@ -455,21 +429,87 @@ describe('a provider process that outlived the crash (R1T-2)', () => { await catchingUp await rig.host.restoreListedFromPerChatFiles(listed) rig.host.seedStoredStatuses(listed) + // Owed: the crash cut its turn, so the settle opens it. + expect(readUnsettledJournalSessionIds(db(rig))).toContain('session-closed') await rig.host.settleOwedSessions(listed) - expect(stopOwnerProcess).toHaveBeenCalled() - expect(rig.store.getRecord('session-closed')!.lease).toMatchObject({ - deathEvidence: { kind: 'pid-absent' } - }) - // Settled from the death evidence: the interrupted turn, not an unconfirmed one. - expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ - lifecycle: 'idle', - summary: { turnOutcome: 'interruption' } - }) - expect(rig.host.hasSession('session-closed')).toBe(false) + expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) + }) + + it("is stopped for an unlisted chat whose lease the settle's own listed restore reconciles", async () => { + const rig = await newRig() + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-open', 'session-closed']) + const listed = listedIds(rig) + expect(listed).toEqual(['session-open']) + const reconcile = vi + .spyOn(rig.store, 'reconcileOnRestart') + .mockRejectedValueOnce(new Error('disk I/O error')) + + // Nothing else reconciles before the settle: its listed restore is the first retry. + await startup(rig, listed) + + expect(reconcile).toHaveBeenCalledTimes(2) + expect(stopOwnerProcess).toHaveBeenCalledTimes(1) + expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) }) }) +/** + * Chats whose turn was running when Orca died, with their provider processes still up: one live + * process the rig's chats share, which a stop ends. The first is listed, the rest have no tab. + */ +async function crashWithLiveOwner( + rig: RestTestRig, + ids: readonly string[] +): Promise> { + const leases = [] + for (const [index, sessionId] of ids.entries()) { + const listed = index === 0 && ids.length > 1 + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) + const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value + await rig.host + .collaboratorsForTests() + .sessions.get(sessionId)! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } + ) + leases.push(rig.store.getRecord(sessionId)!.lease) + } + await rig.crash() + for (const lease of leases) { + await writeOlderBuildLease(rig.root, lease.sessionId, { ...lease }) + } + let alive = true + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + rig.probeOwner.mockImplementation(async () => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + ) + await rig.boot({ stopOwnerProcess }) + return stopOwnerProcess +} + +/** Settled from the death evidence its recovery recorded: the turn reads as interrupted, not as + * unconfirmed, and the chat is closed again. */ +function expectInterruptedAndStopped( + rig: RestTestRig, + sessionId: string, + stopOwnerProcess: ReturnType +): void { + expect(stopOwnerProcess).toHaveBeenCalled() + expect(rig.store.getRecord(sessionId)!.lease).toMatchObject({ + deathEvidence: { kind: 'pid-absent' } + }) + expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ + lifecycle: 'idle', + summary: { turnOutcome: 'interruption' } + }) + expect(rig.host.hasSession(sessionId)).toBe(false) +} + describe('one awaited settle covers a chat whose tab closes meanwhile (R1T-4)', () => { it('settles a listed chat the listed worker skips because its tab closed', async () => { const rig = await newRig() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index f746dd369c5e..fe14568aa0fb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -204,8 +204,13 @@ async function settleOwedSessions( if (deps.isDisposed()) { return } + // A reconcile since the records were read (the listed restore's, or a command's) may have + // moved this lease to `recovering`: recover it, and settle from the record it left. + await leases.resolve(record.sessionId) await deps - .serialize(record.sessionId, () => settleClosed(deps, record)) + .serialize(record.sessionId, () => + settleClosed(deps, deps.openDeps.store.getRecord(record.sessionId) ?? record) + ) .catch((error: unknown) => { deps.openDeps.logger.warn('settling a chat at startup failed', { scope: 'startup-settle-chat', From a53995e664eaeb2be4b1dd50a728dbf746ef98ce Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:38:42 -0700 Subject: [PATCH 087/107] test(native-chat): the closed-chat startup settle tests check the turn's verdict first --- .../structured-agent-session-startup-state.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 1452aa9c7f61..19ec748d9b7d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -499,14 +499,14 @@ function expectInterruptedAndStopped( sessionId: string, stopOwnerProcess: ReturnType ): void { - expect(stopOwnerProcess).toHaveBeenCalled() - expect(rig.store.getRecord(sessionId)!.lease).toMatchObject({ - deathEvidence: { kind: 'pid-absent' } - }) expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle', summary: { turnOutcome: 'interruption' } }) + expect(stopOwnerProcess).toHaveBeenCalled() + expect(rig.store.getRecord(sessionId)!.lease).toMatchObject({ + deathEvidence: { kind: 'pid-absent' } + }) expect(rig.host.hasSession(sessionId)).toBe(false) } From 4550199ab3fdaec47aa94722029648fb2ec7b583 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:39:11 -0700 Subject: [PATCH 088/107] test(native-chat): the batch status write is checked not to rewrite a chat that got its row after its fold Since the batch write stopped returning the chats it wrote, the only check of that skip compared the chat's row with one equal to what its fold would write. The chat's row now differs first, so a rewrite shows. --- .../journal-session-status-backfill.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index 0711c99e70fe..8630c9417514 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -359,7 +359,12 @@ describe('a row from the rows alone', () => { ) dropRow('appended') journalsById.get('rowed')!.backfillSessionStatus() + // Unlike what its fold would write, so a rewrite shows. + database() + .db.prepare('UPDATE journal_session_state SET last_activity_at = 1 WHERE session_id = ?') + .run('rowed') const rowedBefore = readTestJournalSessionStatus(root, 'rowed') + expect(rowedBefore).not.toEqual(folded[2]!.status) publishTestJournalEpoch(database().db, 'replaced', 'epoch-replaced-later') writeJournalSessionStatuses(database(), folded) From 2f291874faad7fcf65c53caf84812ebf883d1cd6 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:47:10 -0700 Subject: [PATCH 089/107] refactor(native-chat): startup checks and recovers chat leases in one phase that every startup restore answers from Startup drove the lease check and lease recovery from five places: the startup step, the catch-up, the pre-listing restore, the settle, and the restore worker's own check. Their order kept producing bugs, most recently a crashed chat with no tab settled without its recovery when the settle's own restore was the call that completed a failed check. Now one phase runs the lease check (once more if it fails) and then starts every recovery. The settle waits for it; the tab listing waits only on the recoveries of the chats it opens. Every startup restore answers its lease check and its recoveries from that phase and never checks the leases itself, so each lease is checked once and recovered once. If the check fails twice, the leases are left to the next attach or send, as before. --- ...tructured-agent-session-restart-restore.ts | 6 +- .../structured-agent-session-reveal.ts | 18 ++-- ...agent-session-startup-lease-phase.test.ts} | 4 +- ...tured-agent-session-startup-lease-phase.ts | 87 +++++++++++++++++++ ...ed-agent-session-startup-lease-recovery.ts | 72 --------------- ...t-session-startup-per-chat-file-restore.ts | 13 +-- ...ctured-agent-session-startup-state.test.ts | 57 +++++++++--- .../structured-agent-session-startup-state.ts | 71 +++++++++------ ...nt-session-startup-status-catch-up.test.ts | 4 +- .../structured-agent-session-startup-step.ts | 4 +- ...-agent-session-startup-tab-restore.test.ts | 46 +++++----- 11 files changed, 222 insertions(+), 160 deletions(-) rename src/main/native-chat/agent-session-wire/{structured-agent-session-startup-lease-recovery.test.ts => structured-agent-session-startup-lease-phase.test.ts} (96%) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 66001d577a07..68b8d6dfb2f7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -96,9 +96,9 @@ export async function restoreStructuredAgentSessionsOnRestart( if (!first) { return } - // One check for the pass. Each chat checks again while it holds, since another writer can mark - // leases unreconciled mid-pass; after the first failure, retrying per chat only waits on the - // same store again, and the next attach or send settles those chats instead. + // One check for the pass, and per chat (startup answers both from its lease phase, which checks + // once); after the first failure, retrying per chat only waits on the same store again, and the + // next attach or send settles those chats instead. let settled = await input.reconcile(first.sessionId) const settleLeases = async (sessionId: string): Promise => { // A session latched in recovery exits here at startup, without waiting for a client. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 53a0656b58b6..9253fbde4b3b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -85,7 +85,6 @@ export function createStructuredAgentSessionHostRestore( seedStatus: StructuredAgentSessionStartupStateDeps['seedStatus'] } ): { - reconcileRestartLeases: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise } & StructuredAgentSessionStartupState { const { reconcileLeases, resolveRecovery, seedStatus, ...rest } = wiring @@ -113,31 +112,24 @@ export function createStructuredAgentSessionHostRestore( openDeps: deps, canSettle, seedStatus, + reconcile, resolveRecovery: readRestore.resolveRecovery, - restoreListed: (records, resolveListedRecovery, concurrency) => - restoreStructuredAgentSessionsOnRestart({ - ...readRestore, - records, - resolveRecovery: resolveListedRecovery, - concurrency - }), + restoreListed: (records, leases, concurrency) => + restoreStructuredAgentSessionsOnRestart({ ...readRestore, ...leases, records, concurrency }), recoveryBudgetMs: deps.startupRecoveryBudgetMs, serialize: rest.serialize, hasSession: rest.hasSession, isListed: readRestore.isListed, isDisposed: rest.isDisposed }) - // Startup's resolver: a lease startup recovered, or gave up on, is not recovered again. + // Startup's lease bookkeeping: no lease is checked again, or recovered twice. const restorer = new StructuredAgentSessionReadableRestorer({ ...readRestore, - resolveRecovery: startup.recoverLease, + ...startup.leases, supportsRecord }) const gate = new StructuredAgentSessionRestartRestoreGate() return { - reconcileRestartLeases: async () => { - await reconcile('startup') - }, // The listed chats the tab list left to it: corrupt, unreadable, or a per-chat file whose open // failed before the listing. restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.test.ts similarity index 96% rename from src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.test.ts rename to src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.test.ts index a70abb49e363..be71be6d5b6a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.test.ts @@ -1,5 +1,5 @@ -// Startup's lease recovery is per lease: the tab listing waits only on the recoveries of the chats -// opened before it, and no lease is recovered twice, whichever restore asks for it. +// Startup's one lease phase: the tab listing waits only on the recoveries of the chats opened before +// it, and no lease is checked again or recovered twice, whichever restore asks for it. import { afterEach, describe, expect, it, vi } from 'vitest' import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts new file mode 100644 index 000000000000..f16057d23027 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts @@ -0,0 +1,87 @@ +// Startup's one lease phase: the restart lease check, then the recovery of every lease a crash left +// `recovering` (a provider process that outlived the crash is stopped and its death recorded), so +// each settle verdict reads that evidence. Every startup restore answers its lease bookkeeping from +// this phase and never runs the check itself; each lease is recovered once, within a budget, so a +// restore waits only on the leases of the chats it opens. + +import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' +import { withTimeout } from '../../../shared/promise-timeout-fallback' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' + +// Record-only (a probe and at most a process stop each), so a few at once. +const RECOVERY_CONCURRENCY = 4 +// A recovery is process probes around at most one stop (its SIGTERM grace is 5.5 s). Past this a +// probe has stopped answering: the lease stays `recovering`, unverified and never called dead, for +// the next attach or send to resolve, and startup goes on. +const RECOVERY_BUDGET_MS = 15_000 + +export type StructuredAgentSessionStartupLeasePhase = { + /** Starts the phase once and answers whether its check settled every lease. Never rejects. */ + reconciled: () => Promise + /** After the check, every lease `recovering` at this call has its recovery started; ends once + * each has ended or outlasted its budget. Never rejects. */ + recovered: () => Promise + /** A startup restore's resolver, after the check. A `recovering` lease answers its one recovery, + * or true once that outlasts its budget: a second beside it could hang the same way, so its chat + * opens unverified. Any other lease is resolved as `resolveRecovery` resolves it. */ + resolve: (sessionId: string) => Promise +} + +export function createStructuredAgentSessionStartupLeasePhase(deps: { + store: Pick + logger: StructuredAgentSessionLogger + /** The restart lease check; false when it failed (reported by it). Never rejects. */ + reconcile: (sessionId: string) => Promise + /** Resolves a `recovering` lease; never rejects. */ + resolveRecovery: (sessionId: string) => Promise + budgetMs?: number +}): StructuredAgentSessionStartupLeasePhase { + const budgetMs = deps.budgetMs ?? RECOVERY_BUDGET_MS + const recoveries = new Map>() + let checking: Promise | null = null + // A failed check is usually a store write that failed once, so it runs once more; past that, the + // next attach or send checks again before it acts. + const reconciled = () => + (checking ??= deps.reconcile('startup').then((settled) => settled || deps.reconcile('startup'))) + const recovering = (sessionId: string) => + deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' + const recoverOnce = (sessionId: string): Promise => { + let recovery = recoveries.get(sessionId) + if (!recovery) { + recovery = withTimeout(deps.resolveRecovery(sessionId), budgetMs, null).then( + (resolved) => { + if (resolved !== null) { + return resolved + } + deps.logger.warn('a chat recovery outlasted startup; left unverified', { + scope: 'startup-recovery-timeout', + sessionId + }) + return true + } + ) + recoveries.set(sessionId, recovery) + } + return recovery + } + return { + reconciled, + recovered: async () => { + await reconciled() + await forEachWithConcurrency( + deps.store.listRecords().filter((record) => record.lease.handoffStage === 'recovering'), + RECOVERY_CONCURRENCY, + async ({ sessionId }) => { + await recoverOnce(sessionId) + } + ) + }, + resolve: async (sessionId) => { + await reconciled() + return recoveries.has(sessionId) || recovering(sessionId) + ? recoverOnce(sessionId) + : deps.resolveRecovery(sessionId) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts deleted file mode 100644 index 6120d486e02a..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-recovery.ts +++ /dev/null @@ -1,72 +0,0 @@ -// Startup's recovery of the leases a crash left `recovering`: a provider process that outlived the -// crash is stopped and its death recorded, so each settle verdict reads that evidence. Each lease is -// recovered once per host, within a budget, by whichever startup phase asks first, so one phase -// waits only on the leases of the chats it opens. - -import { forEachWithConcurrency } from '../../../shared/map-with-concurrency' -import { withTimeout } from '../../../shared/promise-timeout-fallback' -import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' - -// Record-only (a probe and at most a process stop each), so a few at once. -const RECOVERY_CONCURRENCY = 4 -// A recovery is process probes around at most one stop (its SIGTERM grace is 5.5 s). Past this a -// probe has stopped answering: the lease stays `recovering`, unverified and never called dead, for -// the next attach or send to resolve, and startup goes on. -const RECOVERY_BUDGET_MS = 15_000 - -export type StructuredAgentSessionStartupLeaseRecovery = { - /** Starts the recovery of every lease `recovering` at this call, listed or not, and ends once - * each has ended or outlasted its budget; a lease a later reconcile moved there is included the - * next call. Never rejects. */ - recoverAll: () => Promise - /** A restore's resolver. A `recovering` lease answers its one recovery, or true once that - * outlasts its budget: a second beside it could hang the same way, so its chat opens unverified. - * Any other lease is resolved as `resolveRecovery` resolves it. */ - resolve: (sessionId: string) => Promise -} - -export function createStructuredAgentSessionStartupLeaseRecovery( - deps: Pick & { - budgetMs?: number - } -): StructuredAgentSessionStartupLeaseRecovery { - const budgetMs = deps.budgetMs ?? RECOVERY_BUDGET_MS - const recoveries = new Map>() - const recovering = (sessionId: string) => - deps.openDeps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' - const recoverOnce = (sessionId: string): Promise => { - let recovery = recoveries.get(sessionId) - if (!recovery) { - recovery = withTimeout(deps.resolveRecovery(sessionId), budgetMs, null).then( - (resolved) => { - if (resolved !== null) { - return resolved - } - deps.openDeps.logger.warn('a chat recovery outlasted startup; left unverified', { - scope: 'startup-recovery-timeout', - sessionId - }) - return true - } - ) - recoveries.set(sessionId, recovery) - } - return recovery - } - return { - recoverAll: () => - forEachWithConcurrency( - deps.openDeps.store - .listRecords() - .filter((record) => record.lease.handoffStage === 'recovering'), - RECOVERY_CONCURRENCY, - async ({ sessionId }) => { - await recoverOnce(sessionId) - } - ), - resolve: (sessionId) => - recoveries.has(sessionId) || recovering(sessionId) - ? recoverOnce(sessionId) - : deps.resolveRecovery(sessionId) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts index e7cffca64f6c..c8af43340fa6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts @@ -6,17 +6,20 @@ import { readJournalSessionStatuses } from '../agent-session-journal/journal-session-state' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' -import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' +import type { + StructuredAgentSessionStartupLeases, + StructuredAgentSessionStartupStateDeps +} from './structured-agent-session-startup-state' // The listing waits for the whole pass, so a few at once: one slow file or recovery holds one lane. const PRE_LISTING_RESTORE_CONCURRENCY = 4 -/** Never rejects. `recoverLease` resolves each chat's own lease only, so the listing waits on no - * other chat's recovery. */ +/** Never rejects. `leases` resolves each chat's own lease only, so the listing waits on no other + * chat's recovery. */ export async function restoreListedFromPerChatFiles( deps: StructuredAgentSessionStartupStateDeps, listedIds: readonly string[], - recoverLease: (sessionId: string) => Promise + leases: StructuredAgentSessionStartupLeases ): Promise { const database = deps.openDeps.journalDatabase // A newer build's database: the restore after the listing reads every listed chat as before. @@ -37,7 +40,7 @@ export async function restoreListedFromPerChatFiles( : [] }) if (records.length > 0) { - await deps.restoreListed(records, recoverLease, PRE_LISTING_RESTORE_CONCURRENCY) + await deps.restoreListed(records, leases, PRE_LISTING_RESTORE_CONCURRENCY) } } catch (error) { deps.openDeps.logger.warn('restoring chats still in per-chat files at startup failed', { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 19ec748d9b7d..63af888a526f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -413,18 +413,21 @@ describe('a provider process that outlived the crash (R1T-2)', () => { expect(stopOwnerProcess).not.toHaveBeenCalled() }) - it('is stopped for an unlisted chat whose lease the reconcile moves only after the catch-up began', async () => { + it('is stopped for an unlisted chat whose lease a command checks after the lease check failed', async () => { const rig = await newRig() - await restTestChat(rig, 'session-listed', { message: 'done' }) - const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-closed']) + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-listed', 'session-closed']) const listed = listedIds(rig) - vi.spyOn(rig.store, 'reconcileOnRestart').mockRejectedValueOnce(new Error('disk I/O error')) + const failed = new Error('disk I/O error') + vi.spyOn(rig.store, 'reconcileOnRestart') + .mockRejectedValueOnce(failed) + .mockRejectedValueOnce(failed) await rig.host.reconcileRestartLeases() expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) - // A command let through mid-catch-up completes the reconcile, moving the lease to `recovering`. + // An attach let through mid-catch-up checks the leases first, moving them to `recovering`; + // what it answers after that does not matter here. const catchingUp = rig.host.catchUpMissingStatuses(listed) - await rig.host.reconcileRestartLeases() + await restTestChat(rig, 'session-listed').catch(() => undefined) expect(rig.store.getRecord('session-closed')!.lease.handoffStage).toBe('recovering') await catchingUp await rig.host.restoreListedFromPerChatFiles(listed) @@ -436,7 +439,7 @@ describe('a provider process that outlived the crash (R1T-2)', () => { expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) }) - it("is stopped for an unlisted chat whose lease the settle's own listed restore reconciles", async () => { + it("is stopped for an unlisted chat when the lease check's first try fails", async () => { const rig = await newRig() const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-open', 'session-closed']) const listed = listedIds(rig) @@ -445,18 +448,40 @@ describe('a provider process that outlived the crash (R1T-2)', () => { .spyOn(rig.store, 'reconcileOnRestart') .mockRejectedValueOnce(new Error('disk I/O error')) - // Nothing else reconciles before the settle: its listed restore is the first retry. await startup(rig, listed) + // The lease check's second try settles every lease; no restore checks them again. expect(reconcile).toHaveBeenCalledTimes(2) expect(stopOwnerProcess).toHaveBeenCalledTimes(1) expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) }) + + it('leaves every lease to the next attach or send when the lease check fails twice', async () => { + const rig = await newRig() + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-open', 'session-closed']) + const listed = listedIds(rig) + const failed = new Error('disk I/O error') + const reconcile = vi + .spyOn(rig.store, 'reconcileOnRestart') + .mockRejectedValueOnce(failed) + .mockRejectedValueOnce(failed) + + await startup(rig, listed) + + // No startup restore checks the leases again: each chat settles unverified, nothing stopped. + expect(reconcile).toHaveBeenCalledTimes(2) + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle', + summary: { turnOutcome: 'unconfirmed' } + }) + }) }) /** - * Chats whose turn was running when Orca died, with their provider processes still up: one live - * process the rig's chats share, which a stop ends. The first is listed, the rest have no tab. + * Chats whose turn was running when Orca died. The first is listed, the rest have no tab; the last + * one's provider process is still up until a stop ends it. */ async function crashWithLiveOwner( rig: RestTestRig, @@ -485,8 +510,10 @@ async function crashWithLiveOwner( const stopOwnerProcess = vi.fn(() => { alive = false }) - rig.probeOwner.mockImplementation(async () => - alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + rig.probeOwner.mockImplementation(async (record) => + alive && record.sessionId === ids.at(-1) + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } ) await rig.boot({ stopOwnerProcess }) return stopOwnerProcess @@ -712,6 +739,7 @@ describe('a recovery that never answers', () => { openDeps: openDeps as unknown as StructuredAgentSessionStartupStateDeps['openDeps'], canSettle: (record): record is AgentSessionRecord => record !== null, seedStatus: vi.fn(), + reconcile: async () => true, resolveRecovery, restoreListed, serialize: (_sessionId, task) => task(), @@ -724,7 +752,10 @@ describe('a recovery that never answers', () => { await state.settleOwedSessions([]) expect(resolveRecovery).toHaveBeenCalledWith('session-stuck') - expect(restoreListed).toHaveBeenCalledWith([], expect.any(Function)) + expect(restoreListed).toHaveBeenCalledWith([], { + reconcile: expect.any(Function), + resolveRecovery: expect.any(Function) + }) expect(stuck.lease.handoffStage).toBe('recovering') expect(logger.warn).toHaveBeenCalledWith('a chat recovery outlasted startup; left unverified', { scope: 'startup-recovery-timeout', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index fe14568aa0fb..f971f15e9055 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -1,7 +1,8 @@ // What host startup does with each chat's stored status, before any client lists a tab. // -// Recovery: every lease a crash left `recovering` starts resolving first (a surviving provider -// process is stopped and its death recorded), so each settle verdict below reads that evidence. +// Leases: one phase checks them, then starts recovering every lease a crash left `recovering` (a +// surviving provider process is stopped and its death recorded), so each settle verdict below reads +// that evidence; every startup restore answers its lease bookkeeping from that phase. // Catch-up: a listed chat with history here and no stored status yet (the first launch after the // upgrade) gets its row from its rows alone, so what follows reads stored status for it too. A listed // chat whose history is still in a per-chat file is then opened from it, before the listing (see @@ -32,9 +33,9 @@ import { } from './structured-agent-session-conversation-open' import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' import { - createStructuredAgentSessionStartupLeaseRecovery, - type StructuredAgentSessionStartupLeaseRecovery -} from './structured-agent-session-startup-lease-recovery' + createStructuredAgentSessionStartupLeasePhase, + type StructuredAgentSessionStartupLeasePhase +} from './structured-agent-session-startup-lease-phase' import { restoreListedFromPerChatFiles } from './structured-agent-session-startup-per-chat-file-restore' import { catchUpMissingStatuses } from './structured-agent-session-startup-status-catch-up' @@ -48,14 +49,16 @@ export type StructuredAgentSessionStartupStateDeps = { record: AgentSessionRecord, stored: { projected: StructuredAgentSessionStatusProjection; lastActivityAt: number } ) => void + /** The restart lease check; false when it failed (reported by it). Never rejects. */ + reconcile: (sessionId: string) => Promise /** Resolves a `recovering` lease; never throws (a failure is reported and left to the next * attach or send). */ resolveRecovery: (sessionId: string) => Promise - /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish), resolving - * recovery through `resolveRecovery`, at its own concurrency unless given one. */ + /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish), with its + * lease bookkeeping answered by `leases`, at its own concurrency unless given one. */ restoreListed: ( records: AgentSessionRecord[], - resolveRecovery: (sessionId: string) => Promise, + leases: StructuredAgentSessionStartupLeases, concurrency?: number ) => Promise /** Tests shorten it; production takes the default. */ @@ -67,9 +70,17 @@ export type StructuredAgentSessionStartupStateDeps = { isDisposed: () => boolean } +/** A startup restore's lease bookkeeping, answered from the startup lease phase. */ +export type StructuredAgentSessionStartupLeases = { + reconcile: (sessionId: string) => Promise + resolveRecovery: (sessionId: string) => Promise +} + export type StructuredAgentSessionStartupState = { - /** Starts every lease recovery, then writes the stored status of every listed chat with history - * here and none yet, before the listing answers. Never rejects. */ + /** The startup lease phase's check, which then starts every recovery. Never rejects. */ + reconcileRestartLeases: () => Promise + /** Writes the stored status of every listed chat with history here and none yet, before the + * listing answers. Never rejects. */ catchUpMissingStatuses: (listedIds: readonly string[]) => Promise /** Opens listed chats still in per-chat files, before the listing answers. Never rejects. */ restoreListedFromPerChatFiles: (listedIds: readonly string[]) => Promise @@ -77,33 +88,41 @@ export type StructuredAgentSessionStartupState = { seedStoredStatuses: (listedIds: readonly string[]) => string[] /** Settles every chat a gone process left with work, once per host. Never rejects. */ settleOwedSessions: (listedIds: readonly string[]) => Promise - /** Every restart restore's lease resolver, so no `recovering` lease is recovered twice. */ - recoverLease: (sessionId: string) => Promise + /** Every startup restore's lease bookkeeping, so none checks leases or recovers one twice. */ + leases: StructuredAgentSessionStartupLeases } export function createStructuredAgentSessionStartupState( deps: StructuredAgentSessionStartupStateDeps ): StructuredAgentSessionStartupState { - const leases = createStructuredAgentSessionStartupLeaseRecovery({ + const phase = createStructuredAgentSessionStartupLeasePhase({ ...deps, + store: deps.openDeps.store, + logger: deps.openDeps.logger, budgetMs: deps.recoveryBudgetMs }) + const leases: StructuredAgentSessionStartupLeases = { + reconcile: () => phase.reconciled(), + resolveRecovery: phase.resolve + } let settling: Promise | null = null return { - catchUpMissingStatuses: async (listedIds) => { + reconcileRestartLeases: async () => { + await phase.reconciled() + // Started now, awaited by the settle; the listing waits only on its own chats' recoveries. if (!deps.openDeps.journalDatabase.readOnly) { - void leases.recoverAll() + void phase.recovered() } - await catchUpMissingStatuses(deps, listedIds) }, + catchUpMissingStatuses: (listedIds) => catchUpMissingStatuses(deps, listedIds), restoreListedFromPerChatFiles: (listedIds) => - restoreListedFromPerChatFiles(deps, listedIds, leases.resolve), + restoreListedFromPerChatFiles(deps, listedIds, leases), seedStoredStatuses: (listedIds) => seedStoredStatuses(deps, listedIds), settleOwedSessions: (listedIds) => { - settling ??= settleOwedSessions(deps, listedIds, leases) + settling ??= settleOwedSessions(deps, listedIds, phase, leases) return settling }, - recoverLease: leases.resolve + leases } } @@ -168,14 +187,15 @@ function seedStoredStatuses( async function settleOwedSessions( deps: StructuredAgentSessionStartupStateDeps, listedIds: readonly string[], - leases: StructuredAgentSessionStartupLeaseRecovery + phase: StructuredAgentSessionStartupLeasePhase, + leases: StructuredAgentSessionStartupLeases ): Promise { try { const database = deps.openDeps.journalDatabase if (database.readOnly) { return } - await leases.recoverAll() + await phase.recovered() const listedOrder = new Map(listedIds.map((sessionId, index) => [sessionId, index])) const listed: AgentSessionRecord[] = [] const others: AgentSessionRecord[] = [] @@ -195,7 +215,7 @@ async function settleOwedSessions( (left, right) => (listedOrder.get(left.sessionId) ?? 0) - (listedOrder.get(right.sessionId) ?? 0) ) - await deps.restoreListed(listed, leases.resolve) + await deps.restoreListed(listed, leases) // A listed chat the worker left closed (its tab closed meanwhile) is settled like any other. others.push(...listed.filter((record) => !deps.hasSession(record.sessionId))) for (const record of others) { @@ -204,13 +224,8 @@ async function settleOwedSessions( if (deps.isDisposed()) { return } - // A reconcile since the records were read (the listed restore's, or a command's) may have - // moved this lease to `recovering`: recover it, and settle from the record it left. - await leases.resolve(record.sessionId) await deps - .serialize(record.sessionId, () => - settleClosed(deps, deps.openDeps.store.getRecord(record.sessionId) ?? record) - ) + .serialize(record.sessionId, () => settleClosed(deps, record)) .catch((error: unknown) => { deps.openDeps.logger.warn('settling a chat at startup failed', { scope: 'startup-settle-chat', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts index 274dffde1402..a19500408af8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts @@ -124,6 +124,7 @@ function deps( canSettle: (candidate: AgentSessionRecord | null): candidate is AgentSessionRecord => candidate !== null, seedStatus: vi.fn(), + reconcile: async () => true, resolveRecovery: recovering.resolveRecovery, restoreListed: async () => undefined, serialize: (_sessionId: string, task: () => Promise) => task(), @@ -207,7 +208,7 @@ describe('the startup status catch-up', () => { } ) - it('starts every lease recovery, listed or not, before its first fold', async () => { + it('runs after the lease check started every lease recovery, listed or not', async () => { const ids = ['session-0', 'session-1'] await chats(ids, { rowless: true }) const crashed = ['session-0', 'session-tabless'].map((sessionId) => { @@ -226,6 +227,7 @@ describe('the startup status catch-up', () => { ) trace.events = [] + await state.reconcileRestartLeases() await state.catchUpMissingStatuses(ids) const firstFold = trace.events.findIndex((event) => event.startsWith('fold:')) diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index f0d6176f75cb..07d36ba58f6c 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -30,8 +30,8 @@ type StructuredAgentSessionListingHost = Partial< /** * Answers the listed chats stored status could not answer, which the post-listing restore opens. - * The lease check runs first: it marks crashed leases `recovering`, and the recoveries the catch-up - * starts record the death evidence the settle's verdicts read. It reports its own failures and + * The lease check runs first: it marks crashed leases `recovering` and starts their recoveries, + * which record the death evidence the settle's verdicts read. It reports its own failures and * never throws. */ export async function runStructuredAgentSessionStartupStep( diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index 174c277f75df..70be2d1a03e3 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -331,28 +331,32 @@ describe('restoring the chat tabs open at quit', () => { } ) - // Each failed write stands for one refused commit. - it.each([1, 4, 8])('fails one write per startup step with %i chats open', async (count) => { - const chats = Array.from({ length: count }, (_, index) => `chat-${index}-000${index}`) - const records = chats.map((sessionId) => chatRecord(sessionId)) - await seedProfile(records, { visible: chats }) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const { runtime, published } = startupRuntime({ - afterInstall: () => { - writes.failing = true - } - }) + // Each failed write stands for one refused commit. The lease check is tried twice, then left to + // the next attach or send. + it.each([1, 4, 8])( + "fails only the lease check's two writes with %i chats open", + async (count) => { + const chats = Array.from({ length: count }, (_, index) => `chat-${index}-000${index}`) + const records = chats.map((sessionId) => chatRecord(sessionId)) + await seedProfile(records, { visible: chats }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { runtime, published } = startupRuntime({ + afterInstall: () => { + writes.failing = true + } + }) - await runtime.prepareStructuredAgentSessionStartupRestoration() - const prepared = writes.refused - await runtime.restoreStructuredAgentSessionTabs() - await historyRestored(runtime) + await runtime.prepareStructuredAgentSessionStartupRestoration() + const prepared = writes.refused + await runtime.restoreStructuredAgentSessionTabs() + await historyRestored(runtime) - expect(published()).toHaveLength(count) - expect(prepared).toBe(1) - // Every chat's stored state answers it, so the background restore opens none and writes nothing. - expect(writes.refused - prepared).toBe(0) - }) + expect(published()).toHaveLength(count) + expect(prepared).toBe(2) + // Every chat's stored state answers it, so the background restore opens none and writes nothing. + expect(writes.refused - prepared).toBe(0) + } + ) describe('on a legacy profile, with no tab index yet', () => { const legacyChats = () => [ @@ -397,7 +401,7 @@ describe('restoring the chat tabs open at quit', () => { await historyRestored(runtime) expect(published()).toHaveLength(2) - expect(prepared).toBe(1) + expect(prepared).toBe(2) // The seed alone: stored state answers both chats, so the background restore writes nothing. expect(writes.refused - prepared).toBe(1) }) From a2f2ddbc884e0524939ff6554d137d51515cd5f0 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:51:21 -0700 Subject: [PATCH 090/107] fix(native-chat): a stored chat status derived by other rules reads as missing and is derived again Each chat's stored status is a cache of shared derivation code that changes often, and nothing re-derived a row a build with other rules had written: an idle chat kept showing the older build's summary at every launch, and a change to what counts as owed work left crash-cut chats unselected. Each row now records the rules version it was derived by. A row by other rules reads as missing, so it takes the path a chat with no row takes: a listed chat is derived from its rows before the tab listing answers, any other when it opens, and the settle selects only rows by these rules. A test over every corpus chat pins what the derivation writes and fails until the version is bumped with a change to it. The table is not released yet, so the column is part of its creation. --- .../journal-fold-undo.test.ts | 4 +- .../journal-per-session-import.test.ts | 8 +-- .../journal-session-state.ts | 47 ++++++++----- .../journal-session-status-rules.test.ts | 63 ++++++++++++++++++ ...ured-agent-session-status-catch-up.test.ts | 66 +++++++++++++++++++ 5 files changed, 166 insertions(+), 22 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts index a114117ef071..50d90ce4dcf4 100644 --- a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts @@ -225,8 +225,8 @@ describe('a failed append leaves the fold equal to a replay of the disk', () => vi.spyOn(JournalQueuedMessages.prototype, 'onRowInTransaction').mockImplementation((db) => { db.prepare( `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, - queued_sends, live_child_work, summary_json, last_activity_at) - VALUES ('savepoint-write', 'running', NULL, 0, 0, 0, '{}', 0)` + queued_sends, live_child_work, summary_json, last_activity_at, rules_version) + VALUES ('savepoint-write', 'running', NULL, 0, 0, 0, '{}', 0, 1)` ).run() throw new Error('bookkeeping failed') }) diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts index ed6bde5181eb..a46ed689726b 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts @@ -28,7 +28,7 @@ import { } from './journal-host-database-test-support' import { journalDirectoryFor, legacyJournalDatabaseFile } from './journal-paths' import { importPerSessionJournal } from './journal-per-session-import' -import { deriveJournalSessionStatus } from './journal-session-state' +import { deriveJournalSessionStatus, JOURNAL_SESSION_STATUS_RULES } from './journal-session-state' import { readJournalSessionEpoch, type JournalStoredRow } from './journal-row-table' import { createStructuredAgentSessionLogger } from '../agent-session-wire/structured-agent-session-logger' @@ -219,9 +219,9 @@ describe('importing a per-chat journal', () => { const { db } = openTestJournalHostDatabase(root) db.prepare( `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, - queued_sends, live_child_work, summary_json, last_activity_at) - VALUES (?, 'running', NULL, 3, 0, 0, '{"status":"working","latestPrompt":"stale"}', 1)` - ).run(IDENTITY.sessionId) + queued_sends, live_child_work, summary_json, last_activity_at, rules_version) + VALUES (?, 'running', NULL, 3, 0, 0, '{"status":"working","latestPrompt":"stale"}', 1, ?)` + ).run(IDENTITY.sessionId, JOURNAL_SESSION_STATUS_RULES) await importPerSessionJournal({ database: openTestJournalHostDatabase(root), diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index c6fbc3a87ec5..cb49e592a930 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -8,8 +8,9 @@ // and seeds every other chat's status from them. // // The table arrives with schema version 5, so a build older than that opens the database read-only -// and never writes a journal row without its status. Rows carry no rules version: a change to what -// the derivation or the selection reads needs a migration that recreates the table. +// and never writes a journal row without its status. Each row carries the rules it was derived by: +// a row derived by other rules reads as missing, so it is derived again exactly as a chat with no +// row is (a listed chat before the listing answers, any other when it opens). import type Database from '../../sqlite/sync-database' import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' @@ -21,6 +22,11 @@ import { journalSettlementFacts } from './journal-open-settlement-plan' import { replayJournal } from './journal-open' import { renderJournalState, type JournalReducerState } from './journal-reducer' +/** What the stored status is derived by: the derivation below and the shared projection it reads. + * Bump it with any change to what either produces for the same journal; the corpus digest test + * fails until it is bumped. */ +export const JOURNAL_SESSION_STATUS_RULES = 1 + /** Work a gone process can have left: running work, a waiting prompt, unanswered or queued sends, * or live child work. Startup settles exactly these chats. */ const UNSETTLED = `lifecycle <> 'idle' OR handed_over_sends > 0 OR queued_sends > 0 OR live_child_work = 1` @@ -38,7 +44,8 @@ CREATE TABLE journal_session_state ( queued_sends INTEGER NOT NULL, live_child_work INTEGER NOT NULL, summary_json TEXT NOT NULL, - last_activity_at INTEGER NOT NULL + last_activity_at INTEGER NOT NULL, + rules_version INTEGER NOT NULL ); CREATE INDEX journal_session_state_unsettled ON journal_session_state (session_id) WHERE ${UNSETTLED}; @@ -103,13 +110,13 @@ export function isUnsettledJournalSessionStatus(status: JournalSessionStatus): b } const UPSERT_STATUS = `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, - handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?) + handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at, rules_version) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET lifecycle = excluded.lifecycle, active_turn_id = excluded.active_turn_id, handed_over_sends = excluded.handed_over_sends, queued_sends = excluded.queued_sends, live_child_work = excluded.live_child_work, summary_json = excluded.summary_json, - last_activity_at = excluded.last_activity_at` + last_activity_at = excluded.last_activity_at, rules_version = excluded.rules_version` const STATUS_COLUMNS = `st.lifecycle AS lifecycle, st.active_turn_id AS active_turn_id, st.handed_over_sends AS handed_over_sends, st.queued_sends AS queued_sends, st.live_child_work AS live_child_work, st.summary_json AS summary_json, @@ -129,7 +136,8 @@ export function writeJournalSessionStatus( status.queuedSends, status.liveChildWork ? 1 : 0, JSON.stringify(status.summary), - status.lastActivityAt + status.lastActivityAt, + JOURNAL_SESSION_STATUS_RULES ) } @@ -150,15 +158,18 @@ export function deleteJournalSessionStatus(db: Database.Database, sessionId: str db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) } -/** Whether the chat has a status row: an open writes one for a chat an older build last wrote. */ +/** Whether the chat has a status row by these rules: an open writes one for a chat an older build + * last wrote, or wrote by other rules. */ export function hasJournalSessionStatus(db: Database.Database, sessionId: string): boolean { return ( - db.prepare('SELECT 1 FROM journal_session_state WHERE session_id = ?').get(sessionId) !== - undefined + db + .prepare('SELECT 1 FROM journal_session_state WHERE session_id = ? AND rules_version = ?') + .get(sessionId, JOURNAL_SESSION_STATUS_RULES) !== undefined ) } -/** A chat's stored status for the startup seed: null when the chat has a journal but no row. */ +/** A chat's stored status for the startup seed: null when the chat has a journal but no row by + * these rules. */ export type StoredJournalSessionStatus = { sessionId: string status: JournalSessionStatus | null @@ -178,10 +189,11 @@ export function readJournalSessionStatuses( const rows = db .prepare( `SELECT s.session_id AS session_id, ${STATUS_COLUMNS} -FROM journal_sessions s LEFT JOIN journal_session_state st ON st.session_id = s.session_id +FROM journal_sessions s LEFT JOIN journal_session_state st + ON st.session_id = s.session_id AND st.rules_version = ? WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` ) - .all(...chunk) + .all(JOURNAL_SESSION_STATUS_RULES, ...chunk) for (const row of rows) { if (typeof row.session_id === 'string') { results.push({ sessionId: row.session_id, status: parseStatusRow(row) }) @@ -191,11 +203,14 @@ WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` return results } -/** Every chat whose stored status says a gone process left it work, through the partial index. */ +/** Every chat whose stored status by these rules says a gone process left it work, through the + * partial index. */ export function readUnsettledJournalSessionIds(db: Database.Database): string[] { return db - .prepare(`SELECT session_id FROM journal_session_state WHERE ${UNSETTLED}`) - .all() + .prepare( + `SELECT session_id FROM journal_session_state WHERE rules_version = ? AND (${UNSETTLED})` + ) + .all(JOURNAL_SESSION_STATUS_RULES) .flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) } diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts new file mode 100644 index 000000000000..bef7fd41b305 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts @@ -0,0 +1,63 @@ +// The stored status is a cache of its derivation: a row derived by other rules reads as missing and +// is derived again. This pins what the derivation writes for every corpus chat, so a change to it +// (or to the shared projection it reads) fails here until the rules version is bumped with it. + +import { createHash } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { + createTrackedJournalOpener, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import { JOURNAL_SESSION_STATUS_RULES } from './journal-session-state' +import { + CORPUS_FENCE, + JOURNAL_SESSION_STATE_CASES, + JOURNAL_SESSION_STATE_CORPUS +} from './journal-session-state-test-corpus' + +// Recorded with the rules version: change both together, after checking the new rows are right. +const DERIVED = { + rules: 1, + digest: '29251b320d2adb45c62e68fd0025c9e6bc24b2f04f094e49b09e7052c6160a32' +} + +const journals = createTrackedJournalOpener() +const roots: string[] = [] + +afterEach(async () => { + await journals.closeAll() + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +it('derives the same row for every corpus chat as the recorded rules version', async () => { + const rows: Record = {} + for (const name of JOURNAL_SESSION_STATE_CASES) { + const root = await mkdtemp(join(tmpdir(), 'orca-status-rules-')) + roots.push(root) + let clock = 1_000 + const journal = await journals.open({ + identity: { + sessionId: name, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${name}` } + }, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${name}`, + currentFence: () => CORPUS_FENCE + }) + await JOURNAL_SESSION_STATE_CORPUS[name](journal) + rows[name] = readTestJournalSessionStatus(root, name) + } + const digest = createHash('sha256').update(JSON.stringify(rows)).digest('hex') + + expect( + { rules: JOURNAL_SESSION_STATUS_RULES, digest }, + 'The stored status derivation changed: bump JOURNAL_SESSION_STATUS_RULES and record the new digest' + ).toEqual(DERIVED) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts index a578303a5a63..a33515a7174c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts @@ -6,6 +6,7 @@ import { statSync } from 'node:fs' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import type * as JournalSessionStateModule from '../agent-session-journal/journal-session-state' +import { JOURNAL_SESSION_STATUS_RULES } from '../agent-session-journal/journal-session-state' import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' import type * as JournalRecoveryModule from './agent-session-journal-recovery' import type * as PerSessionImportModule from '../agent-session-journal/journal-per-session-import' @@ -559,3 +560,68 @@ describe('listed chats with no stored status after the upgrade', () => { expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) }, 120_000) }) + +describe('a stored status derived by other rules', () => { + /** The chat's row as a build with other derivation rules left it. */ + function deriveByOtherRules(rig: RestTestRig, sessionId: string, lifecycle = 'idle'): void { + openTestJournalHostDatabase(rig.root) + .db.prepare( + `UPDATE journal_session_state SET rules_version = ?, lifecycle = ?, + summary_json = '{"status":"idle","latestPrompt":"by other rules"}' WHERE session_id = ?` + ) + .run(JOURNAL_SESSION_STATUS_RULES - 1, lifecycle, sessionId) + } + + const rulesOf = (rig: RestTestRig, sessionId: string) => + openTestJournalHostDatabase(rig.root) + .db.prepare('SELECT rules_version FROM journal_session_state WHERE session_id = ?') + .get(sessionId)?.rules_version + + it('is derived again for a listed chat exactly as a missing row is, and its old row is never shown', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-kept', { message: 'asked session-kept' }) + await restTestChat(rig, 'session-other-rules', { message: 'asked session-other-rules' }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + deriveByOtherRules(rig, 'session-other-rules') + await rig.boot() + + const background = await startup(rig) + + // The rowless path: one fold, one batched write, no open. + expect(work.folded).toEqual(['session-other-rules']) + expect(work.commits).toBe(1) + expect(restTestOpens(rig, 'session-other-rules')).toBe(0) + expect(background).toEqual([]) + expect(rulesOf(rig, 'session-other-rules')).toBe(JOURNAL_SESSION_STATUS_RULES) + expect(latestRestTestStatus(rig, 'session-other-rules')).toMatchObject({ + status: 'idle', + latestPrompt: 'asked session-other-rules' + }) + }) + + it('is left alone for a chat with no tab until it opens, and never selects it to settle', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-closed', { message: 'asked session-closed', listed: false }) + await rig.host.flushAllStreamedEvents() + await rig.crash() + // By other rules it would be owed: this build does not trust that. + deriveByOtherRules(rig, 'session-closed', 'running') + await rig.boot() + + await startup(rig) + + expect(work.folded).toEqual([]) + expect(restTestOpens(rig, 'session-closed')).toBe(0) + expect(rulesOf(rig, 'session-closed')).toBe(JOURNAL_SESSION_STATUS_RULES - 1) + expect(latestRestTestStatus(rig, 'session-closed')).toBeUndefined() + + // Its open writes the row by these rules. + await rig.host.history({ sessionId: 'session-closed', direction: 'tail' }) + expect(rulesOf(rig, 'session-closed')).toBe(JOURNAL_SESSION_STATUS_RULES) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle', + summary: { latestPrompt: 'asked session-closed' } + }) + }) +}) From 50092dcd11da14037a0e815b65404716d333e6f4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:05:03 -0700 Subject: [PATCH 091/107] test(native-chat): the crashed-chat helper types the leases it collects --- .../structured-agent-session-startup-state.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 63af888a526f..21dfc01c9ae3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -487,7 +487,7 @@ async function crashWithLiveOwner( rig: RestTestRig, ids: readonly string[] ): Promise> { - const leases = [] + const leases: AgentSessionRecord['lease'][] = [] for (const [index, sessionId] of ids.entries()) { const listed = index === 0 && ids.length > 1 await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) From f125822524cc505ccc18d4b87426aa8513d3d361 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:08:47 -0700 Subject: [PATCH 092/107] fix(native-chat): startup's lease answers are read from the records, and each closed chat is recovered just before its settle If the startup lease check failed twice, the lease phase kept that failure as its answer. An attach or send let through by the startup wait could then complete the check, but every startup restore still read the old answer and skipped recovery, and a crashed chat with no tab whose lease moved after the settle listed its recoveries was settled without it: its turn read "Couldn't confirm" and its provider process was never stopped. The phase now answers whether the leases are checked from the records each time, keeping only its recoveries. Each closed chat has its lease recovered just before its settle, which reads the record that recovery left. --- docs/reference/agent-status-store.md | 3 +- ...tured-agent-session-startup-lease-phase.ts | 14 ++-- ...ctured-agent-session-startup-state.test.ts | 65 +++++++++++++++++-- .../structured-agent-session-startup-state.ts | 7 +- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index 754ee3947b86..feefb26e3072 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -132,7 +132,8 @@ Two rules the ingest must keep: is the durable truth for a structured session. Each chat's status is stored there beside its journal (`journal_session_state`), written in the same transaction as the journal rows it describes, so it is always current. A - chat last written before that table existed has no row at first: a listed + chat last written before that table existed, or whose row was derived by + other rules, has no row at first: a listed chat gets one at startup, before the tab listing answers, computed from its journal rows without opening it, so the republish and the settle below cover it too; any other chat gets one when it is opened, which also settles it. A diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts index f16057d23027..5a07bd6945c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.ts @@ -17,7 +17,8 @@ const RECOVERY_CONCURRENCY = 4 const RECOVERY_BUDGET_MS = 15_000 export type StructuredAgentSessionStartupLeasePhase = { - /** Starts the phase once and answers whether its check settled every lease. Never rejects. */ + /** Starts the phase's check once; answers, from the records, whether every lease is settled now. + * Never rejects. */ reconciled: () => Promise /** After the check, every lease `recovering` at this call has its recovery started; ends once * each has ended or outlasted its budget. Never rejects. */ @@ -41,9 +42,14 @@ export function createStructuredAgentSessionStartupLeasePhase(deps: { const recoveries = new Map>() let checking: Promise | null = null // A failed check is usually a store write that failed once, so it runs once more; past that, the - // next attach or send checks again before it acts. - const reconciled = () => - (checking ??= deps.reconcile('startup').then((settled) => settled || deps.reconcile('startup'))) + // leases stay unchecked until an attach or send checks them. The answer is read from the records, + // never kept, so a check a command completes later counts here too. + const reconciled = async () => { + await (checking ??= deps + .reconcile('startup') + .then((settled) => settled || deps.reconcile('startup'))) + return !deps.store.listRecords().some((record) => record.lease.unreconciled) + } const recovering = (sessionId: string) => deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' const recoverOnce = (sessionId: string): Promise => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index 21dfc01c9ae3..f2bdb81ce4ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -37,6 +37,7 @@ import { latestRestTestStatus, restTestOpens } from './structured-agent-session-rest-test-observations' +import { moveRestTestChatToPerChatFile } from './structured-agent-session-rest-test-per-chat-file' const rigs: RestTestRig[] = [] @@ -456,6 +457,59 @@ describe('a provider process that outlived the crash (R1T-2)', () => { expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) }) + it('is stopped for an unlisted chat whose lease a command checks once the settle is running', async () => { + const rig = await newRig() + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-listed', 'session-closed']) + const listed = listedIds(rig) + const failed = new Error('disk I/O error') + const reconcile = vi + .spyOn(rig.store, 'reconcileOnRestart') + .mockRejectedValueOnce(failed) + .mockRejectedValueOnce(failed) + await rig.host.reconcileRestartLeases() + await rig.host.catchUpMissingStatuses(listed) + await rig.host.restoreListedFromPerChatFiles(listed) + rig.host.seedStoredStatuses(listed) + + const settling = rig.host.settleOwedSessions(listed) + // The settle has waited on its recoveries and read its records; an attach the gate let through + // now checks the leases, and what it answers after that does not matter here. + await new Promise((resolve) => setImmediate(resolve)) + expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) + await restTestChat(rig, 'session-listed').catch(() => undefined) + await settling + + expect(reconcile).toHaveBeenCalledTimes(3) + expect(stopOwnerProcess).toHaveBeenCalledTimes(1) + expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) + }) + + it('is stopped for a listed chat still in its per-chat file whose lease a command checks after the lease check failed', async () => { + const rig = await newRig() + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-file', 'session-other'], { + live: 'session-file', + beforeBoot: () => moveRestTestChatToPerChatFile(rig, 'session-file') + }) + const listed = listedIds(rig) + expect(listed).toEqual(['session-file']) + const failed = new Error('disk I/O error') + vi.spyOn(rig.store, 'reconcileOnRestart') + .mockRejectedValueOnce(failed) + .mockRejectedValueOnce(failed) + await rig.host.reconcileRestartLeases() + // An attach let through before the listing checks the leases; its own answer does not matter. + await restTestChat(rig, 'session-other', { listed: false }).catch(() => undefined) + expect(rig.store.getRecord('session-file')!.lease.handoffStage).toBe('recovering') + + await rig.host.catchUpMissingStatuses(listed) + await rig.host.restoreListedFromPerChatFiles(listed) + + // Opened before the listing from its file, after its recovery: the turn reads as interrupted. + expect(rig.host.hasSession('session-file')).toBe(true) + expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ turnOutcome: 'interruption' }) + expect(stopOwnerProcess).toHaveBeenCalledTimes(1) + }) + it('leaves every lease to the next attach or send when the lease check fails twice', async () => { const rig = await newRig() const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-open', 'session-closed']) @@ -480,12 +534,14 @@ describe('a provider process that outlived the crash (R1T-2)', () => { }) /** - * Chats whose turn was running when Orca died. The first is listed, the rest have no tab; the last - * one's provider process is still up until a stop ends it. + * Chats whose turn was running when Orca died. The first is listed, the rest have no tab; the `live` + * one's provider process (the last, unless named) is still up until a stop ends it. `beforeBoot` + * runs between the crash and the next launch. */ async function crashWithLiveOwner( rig: RestTestRig, - ids: readonly string[] + ids: readonly string[], + options: { live?: string; beforeBoot?: () => Promise } = {} ): Promise> { const leases: AgentSessionRecord['lease'][] = [] for (const [index, sessionId] of ids.entries()) { @@ -506,12 +562,13 @@ async function crashWithLiveOwner( for (const lease of leases) { await writeOlderBuildLease(rig.root, lease.sessionId, { ...lease }) } + await options.beforeBoot?.() let alive = true const stopOwnerProcess = vi.fn(() => { alive = false }) rig.probeOwner.mockImplementation(async (record) => - alive && record.sessionId === ids.at(-1) + alive && record.sessionId === (options.live ?? ids.at(-1)) ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index f971f15e9055..cbc8f83e437b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -224,8 +224,13 @@ async function settleOwedSessions( if (deps.isDisposed()) { return } + // A check a command completed since the records were read may have moved this lease to + // `recovering`: recover it, and settle from the record it left. + await phase.resolve(record.sessionId) await deps - .serialize(record.sessionId, () => settleClosed(deps, record)) + .serialize(record.sessionId, () => + settleClosed(deps, deps.openDeps.store.getRecord(record.sessionId) ?? record) + ) .catch((error: unknown) => { deps.openDeps.logger.warn('settling a chat at startup failed', { scope: 'startup-settle-chat', From df470be13b5f761076a7d46d2d68458935a8f5ac Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:25:07 -0700 Subject: [PATCH 093/107] test(native-chat): the host's published status carries no provider child Main removed which provider child is starting from the status wire; this branch builds the row in its own summary builder, so the host feed test now checks the field stays off it. --- .../structured-agent-session-status-feed.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index a65d1333dbe0..b7de96d12413 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -127,6 +127,8 @@ describe('StructuredAgentSessionStatusFeed', () => { if (firstStatus?.type !== 'status') { throw new Error('status publication missing') } + // Which provider child is starting is not on the wire: nothing reads it. + expect(firstStatus.session).not.toHaveProperty('hostExecutionChild') const journalTime = firstStatus.session.updatedAt sessions.get(SESSION)!.child = null feed.publish(SESSION, journal) From 54da015b066e20ebafe24b0b574ef35228f7d170 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:25:14 -0700 Subject: [PATCH 094/107] test(native-chat): the older-schema downgrade baseline is the v1.4.219 release The read-only round trip and the newer-row-kind case ran against an unreleased main build; they now run against the shipped v1.4.219, which is at schema version 3. --- .../agent-session-stop-event-downgrade.unit.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts index dfa96246aa11..065b5dfa8d81 100644 --- a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts +++ b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts @@ -24,10 +24,10 @@ import { importReleaseCheckoutModule, materializeReleaseCheckout } from './relea // make it delete the journal from that row on, so both ride a tombstone it already reads. const BASELINE_REF = 'v1.4.218' const JOURNAL = 'src/main/native-chat/agent-session-journal' -// A main build that shares this one's host database at the schema version before 5. Version 5 added -// each chat's stored status, so this build's database opens read-only there. Once a build at -// version 5 exists, pin it here and assert the writable round trip again. -const OLDER_SCHEMA_BASELINE_REF = '3727100cc9dbcea6201f8a3e506676a3c4b53b18' +// A release that shares this one's host database at a schema version before 5. Version 5 added each +// chat's stored status, so this build's database opens read-only there. Once a release at version +// 5 exists, pin it here and assert the writable round trip again. +const OLDER_SCHEMA_BASELINE_REF = 'v1.4.219' const IDENTITY: AgentSessionJournalIdentity = { sessionId: 'session-downgrade', From 61e512fa3229ee112d3ec67405712df573b9cec4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:42:57 -0700 Subject: [PATCH 095/107] test(native-chat): type the receipt probe's transaction log without a cast --- .../agent-session-journal/journal-append-status-commit.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts b/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts index f852459322e9..70ea0e1963e3 100644 --- a/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts @@ -82,9 +82,10 @@ function ledgerAnswers(): number { /** A send's ledger answer: one row inside the append's transaction, adopted after its COMMIT. */ function ledgerReceipt(): JournalOperationReceipt & { adopted: number; inTransaction: boolean[] } { + const inTransaction: boolean[] = [] const receipt = { adopted: 0, - inTransaction: [] as boolean[], + inTransaction, write: (connection: ReturnType) => { receipt.inTransaction.push(connection.isTransaction) connection.prepare('INSERT INTO ledger_answers VALUES (?)').run('op-1') From 30054248eb3d6b3656efa4a4dfd7971ea9012e36 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:53:34 -0700 Subject: [PATCH 096/107] fix(native-chat): the stored chat status needs no schema version, so a rollback breaks nothing The status table arrived with schema version 5 so that an older build would open the database read-only and never append history behind a status it does not update. Main's #24576 refuses every chat in a database of a newer version, so a rollback to a build with #24576 but not this change would have opened no chat at all. The table is now created at every writable open with no version bump, as the queued-messages table is, and the database stays at version 4: an older build opens it writable and ignores the table. Each row records the history tip it was derived from (epoch, sequence and timestamp of the newest row, read inside the same statement as the write), and every reader (the startup seed, the catch-up, the per-chat-file restore, settle selection and the open's has-row check) treats a row whose tip is not the chat's tip now as missing. So a row an older build wrote history past, or a same-epoch repair regrew to the same sequence, is derived again; a row whose chat is gone or moved is never selected to settle. --- ...journal-database-records-migration.test.ts | 57 +++++--- .../journal-database-schema.ts | 5 +- .../agent-session-journal/journal-database.ts | 29 ++-- .../journal-per-session-import.test.ts | 10 +- .../journal-session-state.test.ts | 6 +- .../journal-session-state.ts | 93 +++++++----- .../journal-session-status-backfill.ts | 15 +- .../journal-session-status-tip.test.ts | 136 ++++++++++++++++++ ...ured-agent-session-status-catch-up.test.ts | 2 +- ...-session-stop-event-downgrade.unit.test.ts | 56 ++++---- 10 files changed, 294 insertions(+), 115 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts b/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts index e8363cec4ced..83c806b36e84 100644 --- a/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-database-records-migration.test.ts @@ -8,14 +8,13 @@ import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import Database from '../../sqlite/sync-database' import { - journalDatabaseMigratesRecords, journalPragmaNumber, NO_LEGACY_JOURNAL_RECORDS, openJournalDatabase, readJournalDatabaseVersion, type JournalLegacyRecordImport } from './journal-database' -import { createJournalTablesSql, JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' +import { createJournalTablesSql } from './journal-database-schema' import { journalDatabasePath } from './journal-host-database' let root: string @@ -79,7 +78,7 @@ const hasTable = (db: Database.Database, name: string): boolean => .get(name) !== undefined describe('the version-4 migration', () => { - it('moves a version-3 database to the current version with its rows and the copied records', () => { + it('moves a version-3 database to 4 with its rows and the copied records', () => { seedVersion3() const copy = importOf('session-a') @@ -89,7 +88,7 @@ describe('the version-4 migration', () => { expect(opened).toMatchObject({ readOnly: false, legacyRecordImportOwed: false }) expect(copy.runs).toBe(1) inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) + expect(journalPragmaNumber(db, 'user_version')).toBe(4) expect(recordIds(db)).toEqual(['session-a']) expect(db.prepare('SELECT epoch FROM journal_sessions').get()).toEqual({ epoch: 'e1' }) }) @@ -114,7 +113,7 @@ describe('the version-4 migration', () => { sql: string, options?: { simple?: boolean } ) { - if (sql === `user_version = ${JOURNAL_DB_SCHEMA_VERSION}`) { + if (sql === 'user_version = 4') { throw new Error('crash before the version is published') } return original.call(this, sql, options) @@ -133,7 +132,7 @@ describe('the version-4 migration', () => { openJournalDatabase(dbPath, retry).db.close() expect(retry.runs).toBe(1) inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) + expect(journalPragmaNumber(db, 'user_version')).toBe(4) expect(recordIds(db)).toEqual(['session-a']) }) }) @@ -152,7 +151,7 @@ describe('the version-4 migration', () => { inspect((db) => expect(journalPragmaNumber(db, 'user_version')).toBe(3)) openJournalDatabase(dbPath, importOf('session-a')).db.close() inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) + expect(journalPragmaNumber(db, 'user_version')).toBe(4) expect(recordIds(db)).toEqual(['new', 'session-a']) }) }) @@ -173,11 +172,11 @@ describe('the version-4 migration', () => { expect(existsSync(dbPath)).toBe(false) }) - it('creates every table on a fresh file at the current version', () => { + it('creates every table on a fresh file at version 4', () => { openJournalDatabase(dbPath, NO_LEGACY_JOURNAL_RECORDS).db.close() inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(JOURNAL_DB_SCHEMA_VERSION) + expect(journalPragmaNumber(db, 'user_version')).toBe(4) for (const table of [ 'journal_rows', 'agent_session_records', @@ -207,30 +206,44 @@ function openAsVersion3Build(path: string): { db: Database.Database; readOnly: b return { db: probe, readOnly: false } } -describe('version 5, each chat status', () => { - it('moves a version-4 database to 5 without copying the records again, with an empty status table', () => { +/** A build at version 4 that knows nothing of the status table: it latches only a higher version. */ +function openAsVersion4Build(path: string): { db: Database.Database; readOnly: boolean } { + const probe = new Database(path) + if (journalPragmaNumber(probe, 'user_version') > 4) { + probe.close() + return { db: new Database(path, { readonly: true, fileMustExist: true }), readOnly: true } + } + return { db: probe, readOnly: false } +} + +describe('each chat status, beside the version-4 tables', () => { + it('adds its table with no version bump, so a version-4 build still opens the database writable', () => { seedVersion3() openJournalDatabase(dbPath, importOf('session-a')).db.close() - inspect((db) => { - db.pragma('user_version = 4') - db.exec('DROP TABLE journal_session_state') - }) - expect(journalDatabaseMigratesRecords(4)).toBe(false) - const again = importOf('session-b') + inspect((db) => db.exec('DROP TABLE journal_session_state')) - openJournalDatabase(dbPath, again).db.close() + openJournalDatabase(dbPath, importOf('session-b')).db.close() - expect(again.runs).toBe(0) inspect((db) => { - expect(journalPragmaNumber(db, 'user_version')).toBe(5) + expect(journalPragmaNumber(db, 'user_version')).toBe(4) expect(hasTable(db, 'journal_session_state')).toBe(true) - expect(recordIds(db)).toEqual(['session-a']) }) + const older = openAsVersion4Build(dbPath) + try { + expect(older.readOnly).toBe(false) + older.db + .prepare( + "INSERT INTO journal_sessions (session_id, workspace_id, epoch) VALUES ('s2', 'ws', 'e')" + ) + .run() + } finally { + older.db.close() + } }) }) describe('a build from before the move', () => { - it('opens a current database read-only, so it never writes beside the records', () => { + it('opens a version-4 database read-only, so it never writes beside the records', () => { openJournalDatabase(dbPath, importOf('session-a')).db.close() const older = openAsVersion3Build(dbPath) diff --git a/src/main/native-chat/agent-session-journal/journal-database-schema.ts b/src/main/native-chat/agent-session-journal/journal-database-schema.ts index c096d7a82d46..00538fe75f9b 100644 --- a/src/main/native-chat/agent-session-journal/journal-database-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-database-schema.ts @@ -12,10 +12,7 @@ /** DB shape version, carried in `PRAGMA user_version`. Independent of the row body version * (`JournalRow.v`): a newer build can change either alone. A newer version is opened read-only here * and its chats are refused as a newer Orca's, so every change stays additive. */ -export const JOURNAL_DB_SCHEMA_VERSION = 5 - -/** The version whose migration copies the chat records file in: "copied" is `user_version >= 4`. */ -export const JOURNAL_DB_RECORDS_VERSION = 4 +export const JOURNAL_DB_SCHEMA_VERSION = 4 /** The first version a release wrote; 1 and 2 only ever came from development builds. */ export const JOURNAL_DB_OLDEST_RELEASED_VERSION = 3 diff --git a/src/main/native-chat/agent-session-journal/journal-database.ts b/src/main/native-chat/agent-session-journal/journal-database.ts index 8209eaf71950..9b1125cc7efa 100644 --- a/src/main/native-chat/agent-session-journal/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-database.ts @@ -11,12 +11,11 @@ import { createAgentSessionRecordTablesSql, createJournalTablesSql, JOURNAL_DB_OLDEST_RELEASED_VERSION, - JOURNAL_DB_RECORDS_VERSION, JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' import { JournalUnreleasedSchemaError } from './journal-open-failure' import { ensureQueuedMessagesTable } from './queued-message-schema' -import { createJournalSessionStatusTable } from './journal-session-state' +import { ensureJournalSessionStatusTable } from './journal-session-state' export const JOURNAL_BUSY_TIMEOUT_MS = 5000 /** Bounds the WAL a checkpoint leaves behind; SQLite truncates it back to this after a reset. */ @@ -55,7 +54,7 @@ export function journalPragmaNumber(db: Database.Database, name: string): number export function journalDatabaseMigratesRecords(stored: number): boolean { return ( stored === 0 || - (stored >= JOURNAL_DB_OLDEST_RELEASED_VERSION && stored < JOURNAL_DB_RECORDS_VERSION) + (stored >= JOURNAL_DB_OLDEST_RELEASED_VERSION && stored < JOURNAL_DB_SCHEMA_VERSION) ) } @@ -89,7 +88,7 @@ export function journalDatabaseHoldsAgentSessions(dbPath: string): boolean | und (table) => tables.has(table) && db.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() !== undefined ) - if (holds || journalPragmaNumber(db, 'user_version') >= JOURNAL_DB_RECORDS_VERSION) { + if (holds || journalPragmaNumber(db, 'user_version') >= JOURNAL_DB_SCHEMA_VERSION) { return holds } return undefined @@ -129,9 +128,10 @@ export function openJournalDatabase( configureJournalPragmas(probe, stored) const legacyRecordImportOwed = migrateJournalSchema(probe, stored, legacyRecords) // Outside `migrateJournalSchema` on purpose: its early return skips a db - // already at the current version, and this table must exist at EVERY + // already at the current version, and these tables must exist at EVERY // writable open with no `user_version` bump (see `ensureQueuedMessagesTable`). ensureQueuedMessagesTable(probe) + ensureJournalSessionStatusTable(probe) hardenSqliteDatabaseFiles(dbPath) transferred = true return { db: probe, readOnly: false, legacyRecordImportOwed } @@ -164,8 +164,6 @@ function configureJournalPragmas(db: Database.Database, stored: number): void { * Table creation, the records copy and the `user_version` bump are ONE transaction. Creating the * tables first left a shaped database still reporting version 0, which an older build does not * latch read-only; and "copied" is `user_version >= 4`, so no other marker can disagree with it. - * Version 5 adds each chat's status table, recreated empty on every run: while the records copy is - * owed the database stays at a released version an older build can write, so no row survives one. * Returns whether the copy is still owed. */ function migrateJournalSchema( @@ -180,18 +178,15 @@ function migrateJournalSchema( if (stored === 0) { db.exec(createJournalTablesSql()) } - createJournalSessionStatusTable(db) - if (stored < JOURNAL_DB_RECORDS_VERSION) { - db.exec(createAgentSessionRecordTablesSql()) - if (legacyRecords.owed) { - // A fresh file still takes a released version, so an older build opens it as one. - if (stored === 0) { - db.pragma(`user_version = ${JOURNAL_DB_OLDEST_RELEASED_VERSION}`) - } - return + db.exec(createAgentSessionRecordTablesSql()) + if (legacyRecords.owed) { + // A fresh file still takes a released version, so an older build opens it as one. + if (stored === 0) { + db.pragma(`user_version = ${JOURNAL_DB_OLDEST_RELEASED_VERSION}`) } - legacyRecords.write(db) + return } + legacyRecords.write(db) db.pragma(`user_version = ${JOURNAL_DB_SCHEMA_VERSION}`) }) return legacyRecords.owed diff --git a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts index 79afa1477cd9..db1c2372b467 100644 --- a/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-per-session-import.test.ts @@ -221,11 +221,15 @@ describe('importing a per-chat journal', () => { const { epoch, rows } = await historyRows() await writeLegacyJournal(epoch, rows) const { db } = openTestJournalHostDatabase(root) + // Named at the tip the copy publishes, so only the copy's own drop can retire it. + const tip = rows.at(-1)! db.prepare( `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, - queued_sends, live_child_work, summary_json, last_activity_at, rules_version) - VALUES (?, 'running', NULL, 3, 0, 0, '{"status":"working","latestPrompt":"stale"}', 1, ?)` - ).run(IDENTITY.sessionId, JOURNAL_SESSION_STATUS_RULES) + queued_sends, live_child_work, summary_json, last_activity_at, rules_version, epoch, tip_seq, + tip_ts) + VALUES (?, 'running', NULL, 3, 0, 0, '{"status":"working","latestPrompt":"stale"}', 1, ?, ?, ?, + ?)` + ).run(IDENTITY.sessionId, JOURNAL_SESSION_STATUS_RULES, epoch, tip.seq, tip.ts) await importPerSessionJournal({ database: openTestJournalHostDatabase(root), diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index c3663b23ee4f..49d90a507151 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -380,10 +380,10 @@ describe('epoch writes carry the status (T10)', () => { expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(epoch) }) - it('arrives with schema version 5', async () => { + it('leaves the schema at version 4, so an older build stays writable', async () => { await write('settled') - expect(JOURNAL_DB_SCHEMA_VERSION).toBe(5) - expect(journalPragmaNumber(db(), 'user_version')).toBe(5) + expect(JOURNAL_DB_SCHEMA_VERSION).toBe(4) + expect(journalPragmaNumber(db(), 'user_version')).toBe(4) }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index d5fa6a74d26f..4a296650ace5 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -7,10 +7,11 @@ // chat's current state. Startup selects the chats a gone process left with work from these rows, // and seeds every other chat's status from them. // -// The table arrives with schema version 5, so a build older than that opens the database read-only -// and never writes a journal row without its status. Each row carries the rules it was derived by: -// a row derived by other rules reads as missing, so it is derived again exactly as a chat with no -// row is (a listed chat before the listing answers, any other when it opens). +// The table needs no schema version: an older build opens the database as before, ignores the table, +// and can append history without a status. So each row records the history tip it was derived from +// and the rules it was derived by, and a row whose tip is not the chat's tip now, or whose rules are +// other, reads as missing: it is derived again exactly as a chat with no row is (a listed chat before +// the listing answers, any other when it opens). import type Database from '../../sqlite/sync-database' import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' @@ -30,12 +31,19 @@ export const JOURNAL_SESSION_STATUS_RULES = 3 * or live child work. Startup settles exactly these chats. */ const UNSETTLED = `lifecycle <> 'idle' OR handed_over_sends > 0 OR queued_sends > 0 OR live_child_work = 1` -/** Recreated, empty, by every migration up to version 5: any row it held may predate writes an - * older build made, and an open writes a missing row back. */ -export function createJournalSessionStatusTable(db: Database.Database): void { +/** A row whose recorded tip is still the chat's tip: no row was written past it or in its place. + * `st` is the status row; each check is one primary-key lookup on `journal_rows`. */ +const AT_TIP = `EXISTS (SELECT 1 FROM journal_sessions live JOIN journal_rows tip + ON tip.session_id = live.session_id AND tip.epoch = live.epoch AND tip.seq = st.tip_seq + WHERE live.session_id = st.session_id AND live.epoch = st.epoch AND tip.ts = st.tip_ts) + AND NOT EXISTS (SELECT 1 FROM journal_rows later WHERE later.session_id = st.session_id + AND later.epoch = st.epoch AND later.seq > st.tip_seq)` + +/** Created idempotently at every writable open, with no `user_version` bump, as + * `ensureQueuedMessagesTable` is: an older build stays writable and ignores it. */ +export function ensureJournalSessionStatusTable(db: Database.Database): void { db.exec(` -DROP TABLE IF EXISTS journal_session_state; -CREATE TABLE journal_session_state ( +CREATE TABLE IF NOT EXISTS journal_session_state ( session_id TEXT PRIMARY KEY, lifecycle TEXT NOT NULL, active_turn_id TEXT, @@ -44,9 +52,12 @@ CREATE TABLE journal_session_state ( live_child_work INTEGER NOT NULL, summary_json TEXT NOT NULL, last_activity_at INTEGER NOT NULL, - rules_version INTEGER NOT NULL + rules_version INTEGER NOT NULL, + epoch TEXT NOT NULL, + tip_seq INTEGER NOT NULL, + tip_ts INTEGER NOT NULL ); -CREATE INDEX journal_session_state_unsettled ON journal_session_state (session_id) +CREATE INDEX IF NOT EXISTS journal_session_state_unsettled ON journal_session_state (session_id) WHERE ${UNSETTLED}; `) } @@ -106,14 +117,19 @@ export function isUnsettledJournalSessionStatus(status: JournalSessionStatus): b ) } +// The tip is read inside the statement, so a row always records the history it was written with. const UPSERT_STATUS = `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, - handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at, rules_version) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + handed_over_sends, queued_sends, live_child_work, summary_json, last_activity_at, rules_version, + epoch, tip_seq, tip_ts) +SELECT s.session_id, ?, ?, ?, ?, ?, ?, ?, ?, s.epoch, r.seq, r.ts +FROM journal_sessions s JOIN journal_rows r ON r.session_id = s.session_id AND r.epoch = s.epoch +WHERE s.session_id = ? ORDER BY r.seq DESC LIMIT 1 ON CONFLICT(session_id) DO UPDATE SET lifecycle = excluded.lifecycle, active_turn_id = excluded.active_turn_id, handed_over_sends = excluded.handed_over_sends, queued_sends = excluded.queued_sends, live_child_work = excluded.live_child_work, summary_json = excluded.summary_json, - last_activity_at = excluded.last_activity_at, rules_version = excluded.rules_version` + last_activity_at = excluded.last_activity_at, rules_version = excluded.rules_version, + epoch = excluded.epoch, tip_seq = excluded.tip_seq, tip_ts = excluded.tip_ts` const STATUS_COLUMNS = `st.lifecycle AS lifecycle, st.active_turn_id AS active_turn_id, st.handed_over_sends AS handed_over_sends, st.queued_sends AS queued_sends, st.live_child_work AS live_child_work, st.summary_json AS summary_json, @@ -125,35 +141,43 @@ export function writeJournalSessionStatus( sessionId: string, status: JournalSessionStatus ): void { - db.prepare(UPSERT_STATUS).run( - sessionId, - status.lifecycle, - status.activeTurnId, - status.handedOverSends, - status.queuedSends, - status.liveChildWork ? 1 : 0, - JSON.stringify(status.summary), - status.lastActivityAt, - JOURNAL_SESSION_STATUS_RULES - ) + const written = db + .prepare(UPSERT_STATUS) + .run( + status.lifecycle, + status.activeTurnId, + status.handedOverSends, + status.queuedSends, + status.liveChildWork ? 1 : 0, + JSON.stringify(status.summary), + status.lastActivityAt, + JOURNAL_SESSION_STATUS_RULES, + sessionId + ) + if (written.changes === 0) { + throw new Error(`no journal history to store the status of ${sessionId} beside`) + } } export function deleteJournalSessionStatus(db: Database.Database, sessionId: string): void { db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) } -/** Whether the chat has a status row by these rules: an open writes one for a chat an older build - * last wrote, or wrote by other rules. */ +/** Whether the chat has a current row: by these rules, at its tip. An open writes one for a chat an + * older build last wrote, or wrote by other rules. */ export function hasJournalSessionStatus(db: Database.Database, sessionId: string): boolean { return ( db - .prepare('SELECT 1 FROM journal_session_state WHERE session_id = ? AND rules_version = ?') + .prepare( + `SELECT 1 FROM journal_session_state st WHERE st.session_id = ? AND st.rules_version = ? + AND ${AT_TIP}` + ) .get(sessionId, JOURNAL_SESSION_STATUS_RULES) !== undefined ) } -/** A chat's stored status for the startup seed: null when the chat has a journal but no row by - * these rules. */ +/** A chat's stored status for the startup seed: null when the chat has a journal but no current + * row. */ export type StoredJournalSessionStatus = { sessionId: string status: JournalSessionStatus | null @@ -174,7 +198,7 @@ export function readJournalSessionStatuses( .prepare( `SELECT s.session_id AS session_id, ${STATUS_COLUMNS} FROM journal_sessions s LEFT JOIN journal_session_state st - ON st.session_id = s.session_id AND st.rules_version = ? + ON st.session_id = s.session_id AND st.rules_version = ? AND ${AT_TIP} WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` ) .all(JOURNAL_SESSION_STATUS_RULES, ...chunk) @@ -187,12 +211,13 @@ WHERE s.session_id IN (${chunk.map(() => '?').join(', ')})` return results } -/** Every chat whose stored status by these rules says a gone process left it work, through the - * partial index. */ +/** Every chat whose current row says a gone process left it work, through the partial index. A row + * left behind by a chat that is gone or has moved on is never selected. */ export function readUnsettledJournalSessionIds(db: Database.Database): string[] { return db .prepare( - `SELECT session_id FROM journal_session_state WHERE rules_version = ? AND (${UNSETTLED})` + `SELECT st.session_id AS session_id FROM journal_session_state st +WHERE st.rules_version = ? AND (${UNSETTLED}) AND ${AT_TIP}` ) .all(JOURNAL_SESSION_STATUS_RULES) .flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index e2cf5dabd3cd..10a954a98136 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -1,10 +1,11 @@ // Writing the status row a chat already in the host's database is missing, without opening it. // -// Version 5 creates the status table empty, so after an upgrade every chat has no row until -// something writes it. Rather than open each chat (its lease, settle and conversation), its rows are -// folded a bounded part per task, as a replay folds them, then the row is derived and written in one -// short transaction, only while the chat is still where the fold read it. Callers fold chats one at a -// time and write a slice of their rows in one transaction. +// After the upgrade that adds the status table, or a launch of an older build that wrote history +// past a chat's row, a chat has no current row until something writes it. Rather than open each +// chat (its lease, settle and conversation), its rows are folded a bounded part per task, as a +// replay folds them, then the row is derived and written in one short transaction, only while the +// chat is still where the fold read it. Callers fold chats one at a time and write a slice of their +// rows in one transaction. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { JournalHostDatabase } from './journal-host-database' @@ -128,8 +129,8 @@ export async function foldJournalSessionStatus( return { sessionId, epoch, tip, load, status } } -/** Every folded chat's row in ONE transaction, each only while it still has no row and its rows are - * where its fold read them; a chat skipped is left to its open. */ +/** Every folded chat's row in ONE transaction, each only while it still has no current row and its + * rows are where its fold read them; a chat skipped is left to its open. */ export function writeJournalSessionStatuses( database: JournalHostDatabase, folded: readonly FoldedJournalSessionStatus[] diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts new file mode 100644 index 000000000000..de0890f37331 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts @@ -0,0 +1,136 @@ +// A stored status names the history tip it was derived from. An older build writes history with no +// status, or deletes a chat, and the table carries no schema version to stop it: a row whose tip is +// not the chat's tip now reads as missing, is derived again, and is never selected to settle. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { + createTrackedJournalOpener, + deleteTestJournalRow, + insertTestJournalRowJson, + liveTestJournalRows, + loadTestJournal, + openTestJournalHostDatabase, + readTestJournalSessionStatus +} from './journal-host-database-test-support' +import { + deriveJournalSessionStatus, + hasJournalSessionStatus, + readUnsettledJournalSessionIds +} from './journal-session-state' +import { CORPUS_FENCE, JOURNAL_SESSION_STATE_CORPUS } from './journal-session-state-test-corpus' +import { codexItem, item, runningTool } from './journal-session-state-test-writes' +import type { AgentSessionJournal } from './journal-store' + +const journals = createTrackedJournalOpener() +let root: string +let clock = 1_000 + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-status-tip-')) + clock = 1_000 +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +function open(sessionId: string): Promise { + return journals.open({ + identity: { + sessionId, + workspaceId: 'ws-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: `thread-${sessionId}` } + }, + stateDirectory: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${sessionId}-${clock}`, + currentFence: () => CORPUS_FENCE + }) +} + +const db = () => openTestJournalHostDatabase(root).db +const stored = (sessionId: string) => readTestJournalSessionStatus(root, sessionId) + +function freshDerivation(sessionId: string) { + return deriveJournalSessionStatus(loadTestJournal(root, sessionId)!.state, { + currentFence: CORPUS_FENCE + }) +} + +/** The stored row as SQLite holds it, every column. */ +function rawRow(sessionId: string): Record { + const row: unknown = db() + .prepare('SELECT * FROM journal_session_state WHERE session_id = ?') + .get(sessionId) + if (typeof row !== 'object' || row === null) { + throw new Error(`no status row for ${sessionId}`) + } + return Object.fromEntries(Object.entries(row)) +} + +/** Puts a row back exactly as it was: what is left when a build that keeps no status writes on. */ +function putBack(row: Record): void { + const columns = Object.keys(row) + db() + .prepare( + `INSERT OR REPLACE INTO journal_session_state (${columns.join(', ')}) +VALUES (${columns.map(() => '?').join(', ')})` + ) + .run(...Object.values(row)) +} + +it('reads a row an older build wrote history past as missing, and derives it again', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS.settled(journal) + const before = rawRow('chat') + // The history an older build appends: a tool starts running, and the row stays as it was. + await item(journal, codexItem('turn-1', 4), runningTool) + await journal.close() + putBack(before) + + expect(stored('chat')).toBeNull() + expect(hasJournalSessionStatus(db(), 'chat')).toBe(false) + expect(readUnsettledJournalSessionIds(db())).toEqual([]) + + const reopened = await open('chat') + reopened.sessionStatus.backfill() + + expect(stored('chat')).toEqual(freshDerivation('chat')) + expect(stored('chat')).toMatchObject({ lifecycle: 'running' }) + expect(readUnsettledJournalSessionIds(db())).toEqual(['chat']) +}) + +it('reads a row as missing when a repair in the same epoch regrew the history to its tip', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + const tip = liveTestJournalRows(db(), 'chat').at(-1)! + expect(stored('chat')).not.toBeNull() + + // An older build's repair dropped the tip and wrote a row at the same sequence, later. + deleteTestJournalRow(db(), 'chat', tip.seq) + insertTestJournalRowJson(db(), 'chat', tip.seq, tip.rowJson, tip.ts + 1) + + expect(stored('chat')).toBeNull() + expect(readUnsettledJournalSessionIds(db())).toEqual([]) +}) + +it('never selects the row of a chat whose history is gone', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + expect(readUnsettledJournalSessionIds(db())).toEqual(['chat']) + + // An older build deleted the chat and knew nothing of its status row. + db().prepare('DELETE FROM journal_rows WHERE session_id = ?').run('chat') + db().prepare('DELETE FROM journal_sessions WHERE session_id = ?').run('chat') + + expect(rawRow('chat')).toMatchObject({ lifecycle: 'running' }) + expect(readUnsettledJournalSessionIds(db())).toEqual([]) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts index 4c1b96635ad6..1107cb69da23 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts @@ -140,7 +140,7 @@ async function newRig(): Promise { return rig } -/** Every chat's row dropped, as version 5 finds a database an older build wrote. */ +/** Every chat's row dropped, as the first launch with the status table finds a database. */ function upgradeToEmptyStatusTable(rig: RestTestRig): void { openTestJournalHostDatabase(rig.root).db.prepare('DELETE FROM journal_session_state').run() } diff --git a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts index 38ec757c3274..6ac4d82cf2e4 100644 --- a/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts +++ b/tests/e2e/cross-version-wire/agent-session-stop-event-downgrade.unit.test.ts @@ -15,20 +15,22 @@ import { createTrackedJournalOpener, insertTestJournalRowJson, liveTestJournalRows, + loadTestJournal, openTestJournalHostDatabase, + readTestJournalSessionStatus, SAVED_BY_NEWER_ORCA } from '../../../src/main/native-chat/agent-session-journal/journal-host-database-test-support' import type { JournalRow } from '../../../src/main/native-chat/agent-session-journal/journal-row-schema' +import { deriveJournalSessionStatus } from '../../../src/main/native-chat/agent-session-journal/journal-session-state' import { importReleaseCheckoutModule, materializeReleaseCheckout } from './release-checkout' // A release that knows neither the Stop event nor the Resume marker: an unknown row kind would // make it delete the journal from that row on, so both ride a tombstone it already reads. const BASELINE_REF = 'v1.4.218' const JOURNAL = 'src/main/native-chat/agent-session-journal' -// A release that shares this one's host database at a schema version before 5. Version 5 added each -// chat's stored status, so this build's database opens read-only there. Once a release at version -// 5 exists, pin it here and assert the writable round trip again. -const OLDER_SCHEMA_BASELINE_REF = 'v1.4.219' +// A main build that shares this one's host database and schema version, so a downgrade to it opens +// the journal writable. No release tag has that database yet; move to the first one that does. +const WRITABLE_BASELINE_REF = '3727100cc9dbcea6201f8a3e506676a3c4b53b18' const IDENTITY: AgentSessionJournalIdentity = { sessionId: 'session-downgrade', @@ -180,7 +182,7 @@ function storedRows(directory: string): string[] { } } -test("an older schema's build reads this build's journal read-only, writes nothing, and the pause survives the round trip", async () => { +test("an older build opens this build's journal writable and appends to it; the pause survives the round trip, and the stale status is derived again", async () => { const directory = mkdtempSync(join(tmpdir(), 'orca-stop-event-writable-downgrade-')) const journals = createTrackedJournalOpener() const itemIds = (journal: Pick) => @@ -197,8 +199,9 @@ test("an older schema's build reads this build's journal read-only, writes nothi ]) await journals.closeAll() const rowsBefore = storedRows(directory) + expect(readTestJournalSessionStatus(directory, IDENTITY.sessionId)).not.toBeNull() - const checkout = await materializeReleaseCheckout(OLDER_SCHEMA_BASELINE_REF) + const checkout = await materializeReleaseCheckout(WRITABLE_BASELINE_REF) const support = await importReleaseCheckoutModule( checkout, `${JOURNAL}/journal-host-database-test-support.ts` @@ -206,21 +209,28 @@ test("an older schema's build reads this build's journal read-only, writes nothi const older = releaseExport<() => OlderOpener>(support, 'createTrackedJournalOpener')() try { const downgraded = await older.open({ identity: IDENTITY, stateDirectory: directory }) - expect(downgraded.isReadOnly).toBe(true) + expect(downgraded.isReadOnly).toBe(false) expect(downgraded.cursor()).toEqual(wrote.cursor) expect(itemIds(downgraded)).toEqual(wrote.items) - await expect( - downgraded.appendItem(item(2), { kind: 'status', text: 'the older build' }, scope) - ).rejects.toThrow() + await downgraded.appendItem(item(2), { kind: 'status', text: 'the older build' }, scope) } finally { await older.closeAll() } - expect(storedRows(directory)).toEqual(rowsBefore) + const rowsAfter = storedRows(directory) + expect(rowsAfter.slice(0, rowsBefore.length)).toEqual(rowsBefore) + expect(rowsAfter).toHaveLength(rowsBefore.length + 1) + // The older build kept no status: the row names a tip that is no longer the chat's, so it reads + // as missing until this build derives it again. + expect(readTestJournalSessionStatus(directory, IDENTITY.sessionId)).toBeNull() - // Upgraded again: the history is as this build left it, and the person's Stop still pauses. + // Upgraded again: the older build's row folds, and the person's Stop still pauses the queue. const upgraded = await journals.open({ identity: IDENTITY, stateDirectory: directory }) - expect(upgraded.cursor()).toEqual(wrote.cursor) - expect(itemIds(upgraded)).toEqual(wrote.items) + upgraded.sessionStatus.backfill() + expect(readTestJournalSessionStatus(directory, IDENTITY.sessionId)).toEqual( + deriveJournalSessionStatus(loadTestJournal(directory, IDENTITY.sessionId)!.state) + ) + expect(upgraded.cursor().sequence).toBe(wrote.cursor.sequence + 1) + expect(itemIds(upgraded)).toEqual([...wrote.items, 'codex:thread-1:turn-1:2']) expect(upgraded.queuedMessages.pauses('host-a').map((pause) => pause.reason)).toEqual([ 'stopped' ]) @@ -232,11 +242,11 @@ test("an older schema's build reads this build's journal read-only, writes nothi // Why a Stop's event cannot have a row kind of its own yet: this build keeps a kind it does not know // and refuses the load as a newer Orca's chat, but a build from before that deletes the journal from -// it when it can write it. -// So a Stop kind ships its reader first and is written once no supported build lacks that reader, or -// is written at a bumped `v`. The pinned older build is at the schema before 5, which opens this -// build's database read-only, so it keeps the row too; a released build at version 5 has this rule. -test("this build keeps a newer build's row kind and refuses the load; an older schema's build keeps it too", async () => { +// it. So a Stop kind ships +// its reader first and is written once no supported build lacks that reader, or is written at a +// bumped `v`. Move the baseline to the first release with this rule, and the older build keeps the +// row too. +test("this build keeps a newer build's row kind and refuses the load; a build before it deletes it", async () => { const directory = mkdtempSync(join(tmpdir(), 'orca-newer-kind-downgrade-')) const journals = createTrackedJournalOpener() const newerKinds = () => storedRows(directory).filter((row) => row.includes('"future-mark"')) @@ -271,20 +281,18 @@ test("this build keeps a newer build's row kind and refuses the load; an older s expect(storedRows(directory)).toEqual(rowsBefore) expect(newerKinds()).toEqual([newer]) - const checkout = await materializeReleaseCheckout(OLDER_SCHEMA_BASELINE_REF) + const checkout = await materializeReleaseCheckout(WRITABLE_BASELINE_REF) const support = await importReleaseCheckoutModule( checkout, `${JOURNAL}/journal-host-database-test-support.ts` ) const older = releaseExport<() => OlderOpener>(support, 'createTrackedJournalOpener')() try { - const downgraded = await older.open({ identity: IDENTITY, stateDirectory: directory }) - expect(downgraded.isReadOnly).toBe(true) + await older.open({ identity: IDENTITY, stateDirectory: directory }) } finally { await older.closeAll() } - expect(newerKinds()).toEqual([newer]) - expect(storedRows(directory)).toEqual(rowsBefore) + expect(newerKinds()).toEqual([]) } finally { await journals.closeAll() rmSync(directory, { recursive: true, force: true }) From 943ec2e6bcbd3a7b3189802c396adb7a94dceb03 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:02:05 -0700 Subject: [PATCH 097/107] test(native-chat): the tip test's raw status row is typed as the row SQLite returns --- .../journal-session-status-tip.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts index de0890f37331..10d31e5866d4 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts @@ -23,6 +23,7 @@ import { import { CORPUS_FENCE, JOURNAL_SESSION_STATE_CORPUS } from './journal-session-state-test-corpus' import { codexItem, item, runningTool } from './journal-session-state-test-writes' import type { AgentSessionJournal } from './journal-store' +import type { SqliteRow } from '../../sqlite/sqlite-statement' const journals = createTrackedJournalOpener() let root: string @@ -64,18 +65,18 @@ function freshDerivation(sessionId: string) { } /** The stored row as SQLite holds it, every column. */ -function rawRow(sessionId: string): Record { - const row: unknown = db() +function rawRow(sessionId: string): SqliteRow { + const row = db() .prepare('SELECT * FROM journal_session_state WHERE session_id = ?') .get(sessionId) - if (typeof row !== 'object' || row === null) { + if (!row) { throw new Error(`no status row for ${sessionId}`) } - return Object.fromEntries(Object.entries(row)) + return row } /** Puts a row back exactly as it was: what is left when a build that keeps no status writes on. */ -function putBack(row: Record): void { +function putBack(row: SqliteRow): void { const columns = Object.keys(row) db() .prepare( From 1ac89193f52cdb2d341a918fcec6a71424a0b361 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:21:30 -0700 Subject: [PATCH 098/107] fix(native-chat): an empty epoch gets no status from the catch-up, and one chat's failed write costs only its row A chat whose epoch holds no row (a quit inside an older build's repair, between its delete and its new epoch) made the status write find no tip and throw, which rolled back the whole catch-up batch: up to 15 other listed chats lost their status before the first paint. Such an epoch is now nothing to fold, so it gets no row and its open founds the epoch afresh, and each chat in a batch is written in its own savepoint, so a failed write leaves only that chat without a row. --- .../journal-session-status-backfill.test.ts | 29 +++++++++++++++++++ .../journal-session-status-backfill.ts | 22 +++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts index 34940324c140..5cc2b4e70cb9 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts @@ -363,4 +363,33 @@ describe('a row from the rows alone', () => { expect(readTestJournalSessionStatus(root, 'rowed')).toEqual(rowedBefore) expect(readTestJournalSessionStatus(root, 'replaced')).toBeNull() }) + + it('folds nothing for an epoch with no row, and one chat whose write fails costs only its own row', async () => { + const ids = ['first', 'refused', 'last'] + for (const sessionId of ids) { + await JOURNAL_SESSION_STATE_CORPUS.settled(await open(sessionId)) + dropRow(sessionId) + } + await journals.closeAll() + // A quit inside an older build's repair: the epoch is published and holds no row. + publishTestJournalEpoch(database().db, 'empty', 'epoch-empty') + expect(await foldJournalSessionStatus(database(), 'empty')).toBeNull() + const folded: FoldedJournalSessionStatus[] = [] + for (const sessionId of ids) { + folded.push((await foldJournalSessionStatus(database(), sessionId))!) + } + // Written as if folded before its rows went: the empty epoch's write finds no tip. + folded.splice(1, 0, { ...folded[0]!, sessionId: 'empty', epoch: 'epoch-empty', tip: 0 }) + database().db + .exec(`CREATE TEMP TRIGGER refuse_status BEFORE INSERT ON main.journal_session_state + WHEN NEW.session_id = 'refused' BEGIN SELECT RAISE(ABORT, 'status write refused'); END`) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + + expect(() => writeJournalSessionStatuses(database(), folded)).not.toThrow() + + expect(readTestJournalSessionStatus(root, 'first')).toEqual(folded[0]!.status) + expect(readTestJournalSessionStatus(root, 'last')).toEqual(folded[3]!.status) + expect(readTestJournalSessionStatus(root, 'empty')).toBeNull() + expect(readTestJournalSessionStatus(root, 'refused')).toBeNull() + }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts index 10a954a98136..b2d48b5bbc83 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts @@ -75,8 +75,8 @@ function foldableJournalSessionEpoch( /** The chat's rows folded a part per task, as a replay folds them, and its status derived. Null when * the chat already has a row, has no epoch in this database (it is still in a per-chat file), the - * database or the chat's rows are a newer build's, its history is damaged, or `signal` aborted, which is checked before each - * part. Writes nothing. */ + * database or the chat's rows are a newer build's, its history is damaged or has no row, or + * `signal` aborted, which is checked before each part. Writes nothing. */ export async function foldJournalSessionStatus( database: JournalHostDatabase, sessionId: string, @@ -92,6 +92,11 @@ export async function foldJournalSessionStatus( return null } const tip = readJournalTip(database.db, sessionId, epoch) + // An epoch with no row (a quit inside an older build's repair) has nothing to fold: its open + // founds the epoch afresh and writes the row. + if (tip === 0) { + return null + } const fold = startJournalRowFold({ sessionId, epoch }) for (let afterSeq = Number.MIN_SAFE_INTEGER; ;) { if (signal?.aborted) { @@ -130,7 +135,7 @@ export async function foldJournalSessionStatus( } /** Every folded chat's row in ONE transaction, each only while it still has no current row and its - * rows are where its fold read them; a chat skipped is left to its open. */ + * rows are where its fold read them; a chat skipped, or whose write fails, is left to its open. */ export function writeJournalSessionStatuses( database: JournalHostDatabase, folded: readonly FoldedJournalSessionStatus[] @@ -148,7 +153,16 @@ export function writeJournalSessionStatuses( ) { continue } - writeJournalSessionStatus(db, sessionId, status) + // Its own savepoint: one chat's failed write costs only that chat's row. + db.exec('SAVEPOINT status_row') + try { + writeJournalSessionStatus(db, sessionId, status) + db.exec('RELEASE status_row') + } catch (error) { + db.exec('ROLLBACK TO status_row') + db.exec('RELEASE status_row') + console.warn('[agent-session-journal] writing a chat status failed', { sessionId, error }) + } } }) } From fe4850eda4d3e807eca3acac4d81a33d2db4636b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:04:36 -0700 Subject: [PATCH 099/107] test(native-chat): the stop-note test's journal identity uses the opaque provider handle --- .../structured-agent-session-stop-note-opened-turn.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts index bd425db0824d..77347fb7b346 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts @@ -1,6 +1,7 @@ // A Stop found with no turn running can wait for one to open and interrupt it. Taken, its note is // the turn the provider says it took; refused, the row stays with the conversation, as before. +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -21,7 +22,7 @@ const IDENTITY: AgentSessionJournalIdentity = { workspaceId: 'workspace-1', hostId: 'host-1', agent: 'codex', - providerHandle: { kind: 'codex', threadId: 'thread-1' } + providerHandle: codexProviderHandle('thread-1') } const TURN = { provider: 'codex' as const, threadId: 'thread-1', turnId: 'turn-1', ordinal: 0 } From e9cd7162f916102525101f1582b672db5afa796e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:04:01 -0700 Subject: [PATCH 100/107] test(native-chat): take main's stop-note test, which now imports the provider handle builder itself --- .../structured-agent-session-stop-note-opened-turn.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts index f5dd2142a144..f8df44ce3073 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-note-opened-turn.test.ts @@ -1,7 +1,6 @@ // A Stop found with no turn running can wait for one to open and interrupt it. Taken, its note is // the turn the provider says it took; refused, the row stays with the conversation, as before. -import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' From c3b53829abf36634bd6524fe129d828d061b9917 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 06:38:26 -0700 Subject: [PATCH 101/107] test(native-chat): the close-verdict test's quit-time reader host names its agents, as main's host deps require --- .../structured-agent-session-close-verdict.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts index 6d508b14f744..af9c55a061a5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-verdict.test.ts @@ -136,6 +136,7 @@ async function settledTurn(reader = host) { async function settledTurnAfterQuit() { const state = hostTestState() const reader = new StructuredAgentSessionHost({ + agents: NO_STRUCTURED_AGENTS, logger: createStructuredAgentSessionLogger(), store: state.store, adapter: adapter(), From 05c59087cd772786a3504d7ae2a8b49211ca8b06 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:39:04 -0700 Subject: [PATCH 102/107] refactor(native-chat): drop the first-launch status catch-up; rowless chats get their status from the restore after the listing No one runs native chat on a build without the stored-status table yet, so the first launch with it needs no machinery of its own. Startup no longer derives the missing rows of listed chats from their rows alone before the tab listing answers (with batched, unsynced writes), and no longer opens listed chats still in per-chat files before it. A listed chat with no current row (written before the table, by an older build, or by other rules) or still in a per-chat file is left to the restore after the listing, as on main: that open writes its row, settles it and publishes its status. Removed with it: the fold-only status path, the tip read only it used, and the restore's per-call concurrency only the pre-listing pass set. Their tests go too. Two guards are re-homed: the rules-mismatch reader case is in the tip test, and the "no second recovery for a lease that outlived startup" case now runs on a rowless chat through the restore after the listing. The startup-state suite shares one startup and crash helper, merges its seed/read parity tests, and runs the lease-check permutations as one table. --- docs/reference/agent-status-store.md | 15 +- ...mobile-session-tabs-reconciliation.test.ts | 27 - .../journal-row-table.ts | 7 - .../journal-session-state.ts | 4 +- .../journal-session-status-backfill.test.ts | 396 ---------- .../journal-session-status-backfill.ts | 168 ---- .../journal-session-status-tip.test.ts | 28 +- .../journal-session-status-writer.ts | 5 +- ...d-agent-session-rest-test-per-chat-file.ts | 46 -- ...uctured-agent-session-rest-test-startup.ts | 34 + ...tructured-agent-session-restart-restore.ts | 12 +- ...ent-session-restored-status-parity.test.ts | 92 --- .../structured-agent-session-reveal.ts | 13 +- ...agent-session-startup-damaged-chat.test.ts | 34 +- ...-agent-session-startup-lease-phase.test.ts | 117 --- ...t-session-startup-per-chat-file-restore.ts | 51 -- ...ctured-agent-session-startup-state.test.ts | 718 +++++++----------- .../structured-agent-session-startup-state.ts | 41 +- ...nt-session-startup-status-catch-up.test.ts | 240 ------ ...d-agent-session-startup-status-catch-up.ts | 125 --- ...ured-agent-session-status-catch-up.test.ts | 608 --------------- ...tore-structured-agent-session-tabs-once.ts | 7 +- ...runtime-structured-session-restore.test.ts | 2 - .../structured-agent-session-startup-step.ts | 6 - ...ctured-agent-session-support-probe.test.ts | 2 - ...-agent-session-tab-listing-startup.test.ts | 174 +---- ...on-tabs-sync-slow-initial-listing.test.tsx | 200 ----- ...ed-session-reopened-tab-close.unit.test.ts | 2 - 28 files changed, 401 insertions(+), 2773 deletions(-) delete mode 100644 src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts delete mode 100644 src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-startup.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts delete mode 100644 src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx diff --git a/docs/reference/agent-status-store.md b/docs/reference/agent-status-store.md index a8cc090f306a..77c503837bb3 100644 --- a/docs/reference/agent-status-store.md +++ b/docs/reference/agent-status-store.md @@ -132,15 +132,12 @@ Two rules the ingest must keep: is the durable truth for a structured session. Each chat's status is stored there beside its journal (`journal_session_state`), written in the same transaction as the journal rows it describes, so it is always current. A - chat last written before that table existed, or whose row was derived by - other rules, has no row at first: a listed - chat gets one at startup, before the tab listing answers, computed from its - journal rows without opening it, so the republish and the settle below cover - it too; any other chat gets one when it is opened, which also settles it. A - listed chat whose history is still in an older build's per-chat file is - opened from that file before the tab listing answers, which publishes its - status, and gets its row with its first write. A corrupt chat gets no row - until its open rebuilds it. + chat last written before that table existed, by an older build, or whose row + was derived by other rules, has no current row: it gets one when it is + opened, which also settles it and publishes its status. A listed one is + opened by the background restore after the tab listing answers, as is a + listed chat whose history is still in an older build's per-chat file. A + corrupt chat gets no row until its open rebuilds it. At host startup, before the tab listing answers, the host republishes the row of each settled chat that is still listed and not already open, without opening it; after that it settles every chat a gone process left with work. diff --git a/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts b/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts index 272b00ca5857..3b4199948487 100644 --- a/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts +++ b/mobile/src/session/use-mobile-session-tabs-reconciliation.test.ts @@ -191,33 +191,6 @@ describe('useMobileSessionTabsReconciliation', () => { vi.useRealTimers() }) - // A host's first answer waits on its startup work; on the first boot after an upgrade that can - // outlast the list's own deadline. The tabs stream, already open, still brings the host's tabs. - it("ends with the host's tabs from the stream when a slow host's first list times out", async () => { - sendRequest.mockImplementationOnce(() => - Promise.reject(new Error('Request timed out: session.tabs.list')) - ) - await mount() - expect(subscribe).toHaveBeenCalledWith( - 'session.tabs.subscribe', - { worktree: 'id:repo::worktree' }, - expect.any(Function) - ) - expect(sendRequest).toHaveBeenCalledTimes(1) - expect(applySessionTabs).not.toHaveBeenCalled() - - // The host finishes its startup work: the stream's first frame is the worktree's whole snapshot. - await emitStream({ type: 'snapshot', snapshotVersion: 7, tabs: ['chat-1'] }) - - expect(consumeAcceptedSessionTabs).toHaveBeenCalledWith( - expect.objectContaining({ tabs: ['chat-1'] }), - ['chat-1'], - 'stream' - ) - // The snapshot also re-arms a list, which now answers at once. - expect(sendRequest).toHaveBeenCalledTimes(2) - }) - it('runs one terminal health sweep and zero tab lists in a certified warm minute', async () => { await mount() await emitStream({ type: 'updated', snapshotVersion: 1, tabs: ['tab-1'] }) diff --git a/src/main/native-chat/agent-session-journal/journal-row-table.ts b/src/main/native-chat/agent-session-journal/journal-row-table.ts index 26552602609c..a558252a8994 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-table.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-table.ts @@ -11,7 +11,6 @@ import { AgentSessionJournalError } from './journal-write-guards' export type JournalStoredRow = { epoch: string; seq: number; ts: number; rowJson: string } const SELECT_EPOCH = 'SELECT epoch FROM journal_sessions WHERE session_id = ?' -const SELECT_TIP = 'SELECT max(seq) AS tip FROM journal_rows WHERE session_id = ? AND epoch = ?' const PUBLISH_SESSION_EPOCH = `INSERT INTO journal_sessions (session_id, workspace_id, epoch) VALUES (?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET @@ -30,12 +29,6 @@ export function readJournalSessionEpoch(db: Database.Database, sessionId: string return typeof epoch === 'string' ? epoch : null } -/** The highest sequence stored under `epoch`, or 0 with none. */ -export function readJournalTip(db: Database.Database, sessionId: string, epoch: string): number { - const tip = db.prepare(SELECT_TIP).get(sessionId, epoch)?.tip - return typeof tip === 'number' ? tip : 0 -} - /** Points the chat at `epoch`. Only an epoch change writes this row. */ export function publishJournalSessionEpoch( db: Database.Database, diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.ts b/src/main/native-chat/agent-session-journal/journal-session-state.ts index 4a296650ace5..4e4ad0fd08f7 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.ts @@ -10,8 +10,8 @@ // The table needs no schema version: an older build opens the database as before, ignores the table, // and can append history without a status. So each row records the history tip it was derived from // and the rules it was derived by, and a row whose tip is not the chat's tip now, or whose rules are -// other, reads as missing: it is derived again exactly as a chat with no row is (a listed chat before -// the listing answers, any other when it opens). +// other, reads as missing: it is derived again exactly as a chat with no row is, when the chat opens +// (a listed one in the background restore after the listing). import type Database from '../../sqlite/sync-database' import { isAgentTurnOutcome } from '../../../shared/agent-turn-outcome' diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts deleted file mode 100644 index b9416e5c0c67..000000000000 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.test.ts +++ /dev/null @@ -1,396 +0,0 @@ -// A missing status row written from the chat's rows alone, with no open: the row an open would -// write, folded a bounded part per task, and nothing written when the chat moved or quit stops it. - -import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' -import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' -import { - closeTestJournalHostDatabase, - createTrackedJournalOpener, - insertTestJournalRowJson, - publishTestJournalEpoch, - openTestJournalHostDatabase, - readTestJournalSessionStatus -} from './journal-host-database-test-support' -import * as JournalOpen from './journal-open' -import { - foldJournalSessionStatus, - writeJournalSessionStatuses, - type FoldedJournalSessionStatus -} from './journal-session-status-backfill' -import { - CORPUS_FENCE, - JOURNAL_SESSION_STATE_CASES, - JOURNAL_SESSION_STATE_CORPUS -} from './journal-session-state-test-corpus' -import type { AgentSessionJournal } from './journal-store' - -vi.mock('./journal-open', async (importOriginal) => { - const actual = await importOriginal() - return { ...actual, startJournalRowFold: vi.fn(actual.startJournalRowFold) } -}) - -const journals = createTrackedJournalOpener() -let root: string -let clock = 1_000 - -function identity(sessionId: string): AgentSessionJournalIdentity { - return { - sessionId, - workspaceId: 'ws-1', - hostId: 'local', - agent: 'codex', - providerHandle: codexProviderHandle(`thread-${sessionId}`) - } -} - -function open(sessionId: string): Promise { - return journals.open({ - identity: identity(sessionId), - stateDirectory: root, - now: () => (clock += 1), - mintEpoch: () => `epoch-${sessionId}-${clock}`, - currentFence: () => CORPUS_FENCE - }) -} - -const database = () => openTestJournalHostDatabase(root) -const dropRow = (sessionId: string) => - database().db.prepare('DELETE FROM journal_session_state WHERE session_id = ?').run(sessionId) - -/** The chat's fold, and its row written as a batch of one; null when the fold declines or the - * write leaves the chat rowless. */ -async function foldAndWrite( - sessionId: string, - options?: Parameters[2] -): Promise { - const folded = await foldJournalSessionStatus(database(), sessionId, options) - if (!folded) { - return null - } - writeJournalSessionStatuses(database(), [folded]) - return readTestJournalSessionStatus(root, sessionId) ? folded : null -} - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'orca-status-backfill-')) - clock = 1_000 -}) - -afterEach(async () => { - vi.restoreAllMocks() - await journals.closeAll() - await rm(root, { recursive: true, force: true }) -}) - -describe('a row from the rows alone', () => { - it.each(JOURNAL_SESSION_STATE_CASES)('equals the row an open writes: %s', async (name) => { - const journal = await open(name) - await JOURNAL_SESSION_STATE_CORPUS[name](journal) - await journals.closeAll() - dropRow(name) - // What an open of the chat writes for a missing row. - const reopened = await open(name) - reopened.sessionStatus.backfill() - const byOpen = readTestJournalSessionStatus(root, name) - await journals.closeAll() - dropRow(name) - - const derived = await foldAndWrite(name) - - expect(derived?.status ?? null).toEqual(byOpen) - expect(readTestJournalSessionStatus(root, name)).toEqual(byOpen) - }) - - it("folds nothing for a chat whose rows are damaged or a newer build's, as its open fails", async () => { - const journal = await open('damaged') - await JOURNAL_SESSION_STATE_CORPUS['working subagent roster'](journal) - const tip = journal.cursor() - await journals.closeAll() - insertTestJournalRowJson(database().db, 'damaged', tip.sequence + 1, '{"not a row"') - dropRow('damaged') - - expect(await foldJournalSessionStatus(database(), 'damaged')).toBeNull() - expect(readTestJournalSessionStatus(root, 'damaged')).toBeNull() - }) - - it("folds nothing from a newer build's database", async () => { - await JOURNAL_SESSION_STATE_CORPUS.settled(await open('newer')) - await journals.closeAll() - dropRow('newer') - database().db.pragma(`user_version = ${JOURNAL_DB_SCHEMA_VERSION + 1}`) - closeTestJournalHostDatabase(root) - expect(database().readOnly).toBe(true) - - expect(await foldJournalSessionStatus(database(), 'newer')).toBeNull() - }) - - it('folds a large chat a bounded part per task', async () => { - const journal = await open('large') - const text = 'x'.repeat(2_000) - for (let index = 0; index < 200; index += 1) { - await journal.appendItem( - { provider: 'orca', clientMessageId: `note-${index}` }, - { kind: 'status', text: `${index} ${text}` }, - { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } - ) - } - await journals.closeAll() - dropRow('large') - // Characters of row JSON folded since the last yield: what one task does. - let sinceYield = 0 - let largest = 0 - const actual = await vi.importActual('./journal-open') - vi.mocked(JournalOpen.startJournalRowFold).mockImplementationOnce((input) => { - const fold = actual.startJournalRowFold(input) - return { - add: (entry) => { - sinceYield += entry.rowJson.length - return fold.add(entry) - }, - finish: fold.finish - } - }) - const budget = 20_000 - - const derived = await foldAndWrite('large', { - batchRows: 64, - batchChars: budget, - yieldTask: async () => { - largest = Math.max(largest, sinceYield) - sinceYield = 0 - } - }) - - largest = Math.max(largest, sinceYield) - expect(derived?.status.lifecycle).toBe('idle') - // 200 rows of about 2 KB: many parts, none over the budget. - expect(largest).toBeGreaterThan(budget / 2) - expect(largest).toBeLessThanOrEqual(budget) - }) - - it('writes nothing when the chat moved during the fold, quit stopped it, or it has a row', async () => { - const journal = await open('moving') - await JOURNAL_SESSION_STATE_CORPUS.settled(journal) - dropRow('moving') - // An append lands between parts: the fold read a tip the chat has left. - let appended = false - const moved = await foldAndWrite('moving', { - batchRows: 1, - yieldTask: async () => { - if (!appended) { - appended = true - dropRow('moving') - await journal.appendItem( - { provider: 'orca', clientMessageId: 'late' }, - { kind: 'status', text: 'late' }, - { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } - ) - dropRow('moving') - } - } - }) - expect(moved).toBeNull() - expect(readTestJournalSessionStatus(root, 'moving')).toBeNull() - - const quit = new AbortController() - const stopped = await foldJournalSessionStatus(database(), 'moving', { - batchRows: 1, - signal: quit.signal, - yieldTask: async () => quit.abort() - }) - expect(stopped).toBeNull() - expect(readTestJournalSessionStatus(root, 'moving')).toBeNull() - - // Still in a per-chat file, or never written: no epoch here. - expect(await foldJournalSessionStatus(database(), 'never-written')).toBeNull() - - await foldAndWrite('moving') - const row = readTestJournalSessionStatus(root, 'moving') - expect(row).not.toBeNull() - // A row already there is never rewritten. - expect(await foldJournalSessionStatus(database(), 'moving')).toBeNull() - }) - - it('writes a whole batch in one transaction, each row the row an open writes', async () => { - const byOpen = new Map() - for (const name of JOURNAL_SESSION_STATE_CASES) { - await JOURNAL_SESSION_STATE_CORPUS[name](await open(name)) - } - await journals.closeAll() - for (const name of JOURNAL_SESSION_STATE_CASES) { - dropRow(name) - ;(await open(name)).sessionStatus.backfill() - byOpen.set(name, readTestJournalSessionStatus(root, name)) - await journals.closeAll() - dropRow(name) - } - const folded: FoldedJournalSessionStatus[] = [] - for (const name of JOURNAL_SESSION_STATE_CASES) { - folded.push((await foldJournalSessionStatus(database(), name))!) - } - const transaction = vi.spyOn(database(), 'transaction') - - writeJournalSessionStatuses(database(), folded) - - expect(transaction).toHaveBeenCalledOnce() - for (const name of JOURNAL_SESSION_STATE_CASES) { - expect(readTestJournalSessionStatus(root, name)).toEqual(byOpen.get(name)) - } - }) - - it('commits a batch unsynced and restores the synced level after, also when the commit fails', async () => { - for (const sessionId of ['first', 'second']) { - await JOURNAL_SESSION_STATE_CORPUS.settled(await open(sessionId)) - } - await journals.closeAll() - const folded: FoldedJournalSessionStatus[] = [] - for (const sessionId of ['first', 'second']) { - dropRow(sessionId) - folded.push((await foldJournalSessionStatus(database(), sessionId))!) - } - const connection = database().db - const level = () => Number(connection.pragma('synchronous', { simple: true })) - const exec = connection.exec.bind(connection) - const atCommit: number[] = [] - let failCommit = false - vi.spyOn(connection, 'exec').mockImplementation((sql) => { - if (sql === 'COMMIT') { - atCommit.push(level()) - if (failCommit) { - throw new Error('disk I/O error') - } - } - exec(sql) - }) - // NORMAL (1) for the batch's commit, FULL (2) for every other commit on the connection. - expect(level()).toBe(2) - - writeJournalSessionStatuses(database(), [folded[0]!]) - expect(readTestJournalSessionStatus(root, 'first')).toEqual(folded[0]!.status) - expect(atCommit).toEqual([1]) - expect(level()).toBe(2) - - failCommit = true - expect(() => writeJournalSessionStatuses(database(), [folded[1]!])).toThrow('disk I/O error') - expect(connection.isTransaction).toBe(false) - expect(level()).toBe(2) - expect(readTestJournalSessionStatus(root, 'second')).toBeNull() - }) - - it("leaves the next commit synced, a chat's own append included, after a slice that landed or failed", async () => { - for (const sessionId of ['first', 'second']) { - await JOURNAL_SESSION_STATE_CORPUS.settled(await open(sessionId)) - } - const live = await open('live') - await JOURNAL_SESSION_STATE_CORPUS.settled(live) - const folded: FoldedJournalSessionStatus[] = [] - for (const sessionId of ['first', 'second']) { - dropRow(sessionId) - folded.push((await foldJournalSessionStatus(database(), sessionId))!) - } - const connection = database().db - const exec = connection.exec.bind(connection) - const atCommit: number[] = [] - let failNextCommit = false - vi.spyOn(connection, 'exec').mockImplementation((sql) => { - if (sql === 'COMMIT') { - atCommit.push(Number(connection.pragma('synchronous', { simple: true }))) - if (failNextCommit) { - failNextCommit = false - throw new Error('disk I/O error') - } - } - exec(sql) - }) - const append = (text: string) => - live.appendItem( - { provider: 'orca', clientMessageId: text }, - { kind: 'status', text }, - { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } - ) - - writeJournalSessionStatuses(database(), [folded[0]!]) - await append('after a slice') - failNextCommit = true - expect(() => writeJournalSessionStatuses(database(), [folded[1]!])).toThrow('disk I/O error') - await append('after a failed slice') - database().transaction((db) => db.prepare('SELECT 1').get()) - - // The slices at NORMAL (1); every commit after them, the chat's appends first, at FULL (2). - expect(atCommit).toEqual([1, 2, 1, 2, 2]) - }) - - it('skips only the chats in a batch that moved, got a row or a new history since their fold', async () => { - const ids = ['steady', 'appended', 'rowed', 'replaced'] - const journalsById = new Map() - for (const sessionId of ids) { - const journal = await open(sessionId) - await JOURNAL_SESSION_STATE_CORPUS.settled(journal) - journalsById.set(sessionId, journal) - dropRow(sessionId) - } - const folded: FoldedJournalSessionStatus[] = [] - for (const sessionId of ids) { - folded.push((await foldJournalSessionStatus(database(), sessionId))!) - } - // Between the fold and the batch's write: a send lands, an open writes the row, a rebuild - // starts a new history. - await journalsById - .get('appended')! - .appendItem( - { provider: 'orca', clientMessageId: 'late' }, - { kind: 'status', text: 'late' }, - { fence: CORPUS_FENCE, turnScope: { kind: 'thread' } } - ) - dropRow('appended') - journalsById.get('rowed')!.sessionStatus.backfill() - // Unlike what its fold would write, so a rewrite shows. - database() - .db.prepare('UPDATE journal_session_state SET last_activity_at = 1 WHERE session_id = ?') - .run('rowed') - const rowedBefore = readTestJournalSessionStatus(root, 'rowed') - expect(rowedBefore).not.toEqual(folded[2]!.status) - publishTestJournalEpoch(database().db, 'replaced', 'epoch-replaced-later') - - writeJournalSessionStatuses(database(), folded) - - expect(readTestJournalSessionStatus(root, 'steady')).toEqual(folded[0]!.status) - expect(readTestJournalSessionStatus(root, 'appended')).toBeNull() - expect(readTestJournalSessionStatus(root, 'rowed')).toEqual(rowedBefore) - expect(readTestJournalSessionStatus(root, 'replaced')).toBeNull() - }) - - it('folds nothing for an epoch with no row, and one chat whose write fails costs only its own row', async () => { - const ids = ['first', 'refused', 'last'] - for (const sessionId of ids) { - await JOURNAL_SESSION_STATE_CORPUS.settled(await open(sessionId)) - dropRow(sessionId) - } - await journals.closeAll() - // A quit inside an older build's repair: the epoch is published and holds no row. - publishTestJournalEpoch(database().db, 'empty', 'epoch-empty') - expect(await foldJournalSessionStatus(database(), 'empty')).toBeNull() - const folded: FoldedJournalSessionStatus[] = [] - for (const sessionId of ids) { - folded.push((await foldJournalSessionStatus(database(), sessionId))!) - } - // Written as if folded before its rows went: the empty epoch's write finds no tip. - folded.splice(1, 0, { ...folded[0]!, sessionId: 'empty', epoch: 'epoch-empty', tip: 0 }) - database().db - .exec(`CREATE TEMP TRIGGER refuse_status BEFORE INSERT ON main.journal_session_state - WHEN NEW.session_id = 'refused' BEGIN SELECT RAISE(ABORT, 'status write refused'); END`) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - - expect(() => writeJournalSessionStatuses(database(), folded)).not.toThrow() - - expect(readTestJournalSessionStatus(root, 'first')).toEqual(folded[0]!.status) - expect(readTestJournalSessionStatus(root, 'last')).toEqual(folded[3]!.status) - expect(readTestJournalSessionStatus(root, 'empty')).toBeNull() - expect(readTestJournalSessionStatus(root, 'refused')).toBeNull() - }) -}) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts b/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts deleted file mode 100644 index b2d48b5bbc83..000000000000 --- a/src/main/native-chat/agent-session-journal/journal-session-status-backfill.ts +++ /dev/null @@ -1,168 +0,0 @@ -// Writing the status row a chat already in the host's database is missing, without opening it. -// -// After the upgrade that adds the status table, or a launch of an older build that wrote history -// past a chat's row, a chat has no current row until something writes it. Rather than open each -// chat (its lease, settle and conversation), its rows are folded a bounded part per task, as a -// replay folds them, then the row is derived and written in one short transaction, only while the -// chat is still where the fold read it. Callers fold chats one at a time and write a slice of their -// rows in one transaction. - -import { setImmediate as yieldToEventLoop } from 'node:timers/promises' -import type { JournalHostDatabase } from './journal-host-database' -import { startJournalRowFold, type JournalLoad } from './journal-open' -import { IMPORT_BATCH_ROWS } from './journal-per-session-source' -import { readJournalRowsAfter, readJournalSessionEpoch, readJournalTip } from './journal-row-table' -import { - deriveJournalSessionStatus, - hasJournalSessionStatus, - writeJournalSessionStatus, - type JournalSessionStatus -} from './journal-session-state' - -/** Row JSON per fold part, in UTF-16 units: a page of large rows is split, so each main-thread task - * handles about the same bytes. */ -const FOLD_PART_CHARS = 256 * 1024 - -/** The rows split so no part holds more than `maxChars` of row JSON; a larger row is a part alone. */ -function* charBoundedParts( - rows: readonly Row[], - maxChars: number -): Generator<{ rows: Row[]; last: boolean }> { - let part: Row[] = [] - let chars = 0 - for (const row of rows) { - if (part.length > 0 && chars + row.rowJson.length > maxChars) { - yield { rows: part, last: false } - part = [] - chars = 0 - } - part.push(row) - chars += row.rowJson.length - } - yield { rows: part, last: true } -} - -/** A chat's status folded from its rows, and where the fold read them. */ -export type FoldedJournalSessionStatus = { - sessionId: string - epoch: string - tip: number - load: JournalLoad - status: JournalSessionStatus -} - -type FoldOptions = { - batchRows?: number - batchChars?: number - /** Ends each part's task: the next macrotask by default; a caller can yield only when due. */ - yieldTask?: () => Promise - /** Quit: the fold stops within one part and nothing is written. */ - signal?: AbortSignal -} - -/** The epoch a fold of the chat would read, or null when it has nothing to fold: the database is a - * newer build's, the chat has no epoch here (it is still in a per-chat file), or it has a row. */ -function foldableJournalSessionEpoch( - database: JournalHostDatabase, - sessionId: string -): string | null { - if (database.readOnly) { - return null - } - const epoch = readJournalSessionEpoch(database.db, sessionId) - return epoch === null || hasJournalSessionStatus(database.db, sessionId) ? null : epoch -} - -/** The chat's rows folded a part per task, as a replay folds them, and its status derived. Null when - * the chat already has a row, has no epoch in this database (it is still in a per-chat file), the - * database or the chat's rows are a newer build's, its history is damaged or has no row, or - * `signal` aborted, which is checked before each part. Writes nothing. */ -export async function foldJournalSessionStatus( - database: JournalHostDatabase, - sessionId: string, - { - batchRows = IMPORT_BATCH_ROWS, - batchChars = FOLD_PART_CHARS, - yieldTask = () => yieldToEventLoop(), - signal - }: FoldOptions = {} -): Promise { - const epoch = foldableJournalSessionEpoch(database, sessionId) - if (epoch === null) { - return null - } - const tip = readJournalTip(database.db, sessionId, epoch) - // An epoch with no row (a quit inside an older build's repair) has nothing to fold: its open - // founds the epoch afresh and writes the row. - if (tip === 0) { - return null - } - const fold = startJournalRowFold({ sessionId, epoch }) - for (let afterSeq = Number.MIN_SAFE_INTEGER; ;) { - if (signal?.aborted) { - return null - } - const rows = readJournalRowsAfter(database.db, sessionId, epoch, afterSeq, batchRows) - const last = rows.at(-1) - let folding = true - for (const part of charBoundedParts(rows, batchChars)) { - folding = part.rows.every(fold.add) - // A part per task: a long chat's whole replay in one task holds up every other chat. - if (!folding || part.last) { - break - } - await yieldTask() - if (signal?.aborted) { - return null - } - } - if (!folding || rows.length < batchRows || !last) { - break - } - afterSeq = last.seq - await yieldTask() - } - if (signal?.aborted) { - return null - } - const load = fold.finish() - // A newer build's rows or damage: the chat's open fails, so this build writes nothing for it. - if (load.newer || load.damage) { - return null - } - const status = deriveJournalSessionStatus(load.state) - return { sessionId, epoch, tip, load, status } -} - -/** Every folded chat's row in ONE transaction, each only while it still has no current row and its - * rows are where its fold read them; a chat skipped, or whose write fails, is left to its open. */ -export function writeJournalSessionStatuses( - database: JournalHostDatabase, - folded: readonly FoldedJournalSessionStatus[] -): void { - if (folded.length === 0) { - return - } - // Unsynced: a lost row is re-derived from the journal, and any later synced commit covers it. - database.unsyncedTransaction((db) => { - for (const { sessionId, epoch, tip, status } of folded) { - if ( - hasJournalSessionStatus(db, sessionId) || - readJournalSessionEpoch(db, sessionId) !== epoch || - readJournalTip(db, sessionId, epoch) !== tip - ) { - continue - } - // Its own savepoint: one chat's failed write costs only that chat's row. - db.exec('SAVEPOINT status_row') - try { - writeJournalSessionStatus(db, sessionId, status) - db.exec('RELEASE status_row') - } catch (error) { - db.exec('ROLLBACK TO status_row') - db.exec('RELEASE status_row') - console.warn('[agent-session-journal] writing a chat status failed', { sessionId, error }) - } - } - }) -} diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts index e7b7761bd732..118abed8c036 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts @@ -1,6 +1,7 @@ -// A stored status names the history tip it was derived from. An older build writes history with no -// status, or deletes a chat, and the table carries no schema version to stop it: a row whose tip is -// not the chat's tip now reads as missing, is derived again, and is never selected to settle. +// A stored status names the history tip it was derived from and the rules it was derived by. An +// older build writes history with no status, or deletes a chat, and the table carries no schema +// version to stop it: a row whose tip is not the chat's tip now, or whose rules are other, reads as +// missing, is derived again, and is never selected to settle. import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { mkdtemp, rm } from 'node:fs/promises' @@ -19,6 +20,7 @@ import { import { deriveJournalSessionStatus, hasJournalSessionStatus, + JOURNAL_SESSION_STATUS_RULES, readUnsettledJournalSessionIds } from './journal-session-state' import { CORPUS_FENCE, JOURNAL_SESSION_STATE_CORPUS } from './journal-session-state-test-corpus' @@ -136,3 +138,23 @@ it('never selects the row of a chat whose history is gone', async () => { expect(rawRow('chat')).toMatchObject({ lifecycle: 'running' }) expect(readUnsettledJournalSessionIds(db())).toEqual([]) }) + +it('reads a row derived by other rules as missing, never selects it, and derives it again', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + db() + .prepare('UPDATE journal_session_state SET rules_version = ? WHERE session_id = ?') + .run(JOURNAL_SESSION_STATUS_RULES - 1, 'chat') + + expect(stored('chat')).toBeNull() + expect(hasJournalSessionStatus(db(), 'chat')).toBe(false) + expect(readUnsettledJournalSessionIds(db())).toEqual([]) + + const reopened = await open('chat') + reopened.sessionStatus.backfill() + + expect(rawRow('chat')).toMatchObject({ rules_version: JOURNAL_SESSION_STATUS_RULES }) + expect(stored('chat')).toEqual(freshDerivation('chat')) + expect(readUnsettledJournalSessionIds(db())).toEqual(['chat']) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-writer.ts b/src/main/native-chat/agent-session-journal/journal-session-status-writer.ts index 61076c30e269..5e2f238c6a85 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-writer.ts @@ -127,8 +127,9 @@ export class JournalSessionStatusWriter { } } - /** Writes the chat's status if it has none: a chat an older build last wrote. Bookkeeping: a - * failure leaves the chat without a row, which its next open writes again. */ + /** Writes the chat's status if it has no current row: one last written before stored status, by + * an older build or by other rules. Bookkeeping: a failure leaves the chat without a row, which + * its next open writes again. */ backfill = (): void => { const { host } = this const database = host.database() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts deleted file mode 100644 index fb3adf434697..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file.ts +++ /dev/null @@ -1,46 +0,0 @@ -// Moves a rest rig chat's history out of the host's database into its own per-chat file, as a -// build before the shared database left it: the format an upgrade finds before the copy. - -import { mkdir } from 'node:fs/promises' -import { dirname } from 'node:path' -import Database from '../../sqlite/sync-database' -import { - liveTestJournalRows, - openTestJournalHostDatabase -} from '../agent-session-journal/journal-host-database-test-support' -import { legacyJournalDatabaseFile } from '../agent-session-journal/journal-paths' -import type { RestTestRig } from './structured-agent-session-rest-test-rig' - -export async function moveRestTestChatToPerChatFile( - rig: RestTestRig, - sessionId: string -): Promise { - const host = openTestJournalHostDatabase(rig.root) - const rows = liveTestJournalRows(host.db, sessionId) - const workspaceId = rig.store.getRecord(sessionId)!.location.workspaceId - const path = legacyJournalDatabaseFile(host.legacyDirectoryFor({ workspaceId, sessionId })) - await mkdir(dirname(path), { recursive: true }) - const file = new Database(path) - try { - file.pragma('journal_mode = WAL') - file.exec(` -CREATE TABLE journal_rows (session_id TEXT NOT NULL, epoch TEXT NOT NULL, seq INTEGER NOT NULL, - ts INTEGER NOT NULL, row_json TEXT NOT NULL, PRIMARY KEY (session_id, epoch, seq)); -CREATE TABLE journal_sessions (session_id TEXT PRIMARY KEY, epoch TEXT NOT NULL, updated_at INTEGER NOT NULL); -CREATE TABLE journal_repairs (session_id TEXT PRIMARY KEY, epoch TEXT NOT NULL, - content_from INTEGER NOT NULL, repaired_at INTEGER NOT NULL);`) - file.pragma('user_version = 2') - const insert = file.prepare( - 'INSERT INTO journal_rows (session_id, epoch, seq, ts, row_json) VALUES (?, ?, ?, ?, ?)' - ) - for (const row of rows) { - insert.run(sessionId, row.epoch, row.seq, row.ts, row.rowJson) - } - file.prepare('INSERT INTO journal_sessions VALUES (?, ?, ?)').run(sessionId, rows[0]!.epoch, 1) - } finally { - file.close() - } - for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { - host.db.prepare(`DELETE FROM ${table} WHERE session_id = ?`).run(sessionId) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-startup.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-startup.ts new file mode 100644 index 000000000000..d1b03346ca77 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-startup.ts @@ -0,0 +1,34 @@ +// Host startup and a crash mid-turn on a rest rig, for the startup tests. + +import { restTestChat, type RestTestRig } from './structured-agent-session-rest-test-rig' + +/** What host startup runs, in order, on the ids the tab list names; answers the listed chats it + * left to the restore after the listing. */ +export async function runRestTestStartup( + rig: RestTestRig, + listed: readonly string[] = rig.store.getVisibleSessionTabIndex().sessionIds +): Promise { + await rig.host.reconcileRestartLeases() + const background = rig.host.startup.seedStoredStatuses(listed) + await rig.host.startup.settleOwedSessions(listed) + await rig.host.restoreReadableSessions(background) + return background +} + +/** A chat whose turn is running when Orca dies: startup selects it to settle. */ +export async function crashRestTestChatMidTurn( + rig: RestTestRig, + sessionId: string, + options: { listed?: boolean } = {} +): Promise { + await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, ...options }) + const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value + await rig.host + .collaboratorsForTests() + .sessions.get(sessionId)! + .journal.appendItem( + { ...providerIdentity, ordinal: 0 }, + { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, + { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 68b8d6dfb2f7..27124708c66f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -21,9 +21,8 @@ import type { } from './structured-agent-session-conversation-open' import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' -// One chat at a time after the listing: the restore is CPU-bound on the main thread, so more lanes -// only lengthen each event-loop turn that a user's read or send waits behind. The pass before the -// listing sets its own. +// One chat at a time: the restore is CPU-bound on the main thread, so more lanes only lengthen each +// event-loop turn that a user's read or send waits behind. const JOURNAL_RESTORE_CONCURRENCY = 1 export type StructuredAgentSessionReadRestoreDeps = { @@ -87,10 +86,7 @@ async function restoreOneStructuredAgentSessionReadUnderSerialize( } export async function restoreStructuredAgentSessionsOnRestart( - input: StructuredAgentSessionReadRestoreDeps & { - records: AgentSessionRecord[] - concurrency?: number - } + input: StructuredAgentSessionReadRestoreDeps & { records: AgentSessionRecord[] } ): Promise { const [first] = input.records if (!first) { @@ -111,7 +107,7 @@ export async function restoreStructuredAgentSessionsOnRestart( } const results = await mapSettledWithConcurrency( input.records, - input.concurrency ?? JOURNAL_RESTORE_CONCURRENCY, + JOURNAL_RESTORE_CONCURRENCY, async ({ sessionId }) => { // A journal open is synchronous SQLite: without a macrotask per chat the restore is one long task. await yieldToEventLoop() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts deleted file mode 100644 index d138ee459c28..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restored-status-parity.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -// Status rows now arrive after the tab list, from the post-listing restore pass. Each must end -// exactly where a fresh host lands when the same chat is opened directly by a read (T8, regression -// guard: the pass is main's own open path; this pins that moving it after the list changed nothing). - -import { cp } from 'node:fs/promises' -import { afterEach, expect, it, vi } from 'vitest' -import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' -import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' -import { - createRestTestRig, - restTestChat, - type RestTestRig -} from './structured-agent-session-rest-test-rig' -import { latestRestTestStatus } from './structured-agent-session-rest-test-observations' - -const rigs: RestTestRig[] = [] - -afterEach(async () => { - for (const rig of rigs.splice(0)) { - await rig.dispose() - } - closeTestJournalHostDatabases() - vi.restoreAllMocks() -}) - -const CORPUS = ['session-settled', 'session-quiet', 'session-crashed', 'session-pending'] - -/** The chats a restart finds: two a clean quit settled, one cut off mid-turn by a crash, and one - * whose send the provider only admitted, left pending below a fence that has since moved. */ -async function seedCorpus(rig: RestTestRig): Promise { - await restTestChat(rig, 'session-settled', { message: 'finished work' }) - await restTestChat(rig, 'session-quiet') - await rig.host.flushAllStreamedEvents() - await rig.boot() - await restTestChat(rig, 'session-crashed', { message: 'cut off' }) - rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) - await restTestChat(rig, 'session-pending', { message: 'only admitted' }) - await rig.crash() - // A live lease names the link proven at its fence, so the fence moves with that link. - await rig.store.transitionHandoff('session-pending', (record) => ({ - ...record, - lease: { ...record.lease, runtimeFence: record.lease.runtimeFence + 1 }, - providerHandleChain: record.providerHandleChain.map((link, index, chain) => - index === chain.length - 1 ? { ...link, mintedAtFence: link.mintedAtFence + 1 } : link - ) - })) -} - -it('ends every chat where a direct read of it lands, and shows no pre-crash work (T8)', async () => { - const passRig = await createRestTestRig() - rigs.push(passRig) - await seedCorpus(passRig) - // One copy of the files for each boot, taken while nothing is writing. - closeTestJournalHostDatabases() - const readRoot = `${passRig.root}-read` - await cp(passRig.root, readRoot, { recursive: true }) - const readRig = await createRestTestRig({}, { root: readRoot }) - rigs.push(readRig) - - // Startup's shape: the lease check, the list, then the pass. - const passHost = await passRig.boot() - await passHost.reconcileRestartLeases() - await passHost.restoreReadableSessions(CORPUS) - - // The same run, where the window reads each chat instead. - const readHost = readRig.host - await readHost.reconcileRestartLeases() - for (const sessionId of CORPUS) { - await readHost.history({ sessionId, direction: 'tail' }) - } - - // `updatedAt` is when the row was projected, a wall-clock read that differs between two boots. - const rows = (rig: RestTestRig): Record> => - Object.fromEntries( - CORPUS.flatMap((sessionId) => { - const summary = latestRestTestStatus(rig, sessionId) - if (!summary) { - return [] - } - const { updatedAt: _updatedAt, ...row } = summary - return [[sessionId, row]] - }) - ) - await vi.waitFor(() => expect(rows(passRig)).toEqual(rows(readRig))) - for (const sessionId of CORPUS) { - expect(latestRestTestStatus(passRig, sessionId)).toBeDefined() - } - const crashedRows = passRig.statusEvents.flatMap((event) => - event.type === 'status' && event.session.sessionId === 'session-crashed' ? [event.session] : [] - ) - expect(crashedRows.map((row) => row.status)).not.toContain('working') -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 4bb21154062b..56a49f452d19 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -63,9 +63,8 @@ export async function revealStructuredAgentSession( } } -/** The host's startup restore: reconcile, catch up, open listed chats still in per-chat files, then - * seed and settle from the state stored beside each journal, then open in the background what that - * state cannot answer. Its lease bookkeeping is a +/** The host's startup restore: reconcile, then seed and settle from the state stored beside each + * journal, then open in the background what that state cannot answer. Its lease bookkeeping is a * reader's, which never fails a read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, @@ -107,8 +106,8 @@ export function createStructuredAgentSessionHostRestore( seedStatus, reconcile, resolveRecovery: readRestore.resolveRecovery, - restoreListed: (records, leases, concurrency) => - restoreStructuredAgentSessionsOnRestart({ ...readRestore, ...leases, records, concurrency }), + restoreListed: (records, leases) => + restoreStructuredAgentSessionsOnRestart({ ...readRestore, ...leases, records }), recoveryBudgetMs: deps.startupRecoveryBudgetMs, serialize: rest.serialize, hasSession: rest.hasSession, @@ -122,8 +121,8 @@ export function createStructuredAgentSessionHostRestore( }) const gate = new StructuredAgentSessionRestartRestoreGate() return { - // The listed chats the tab list left to it: corrupt, unreadable, or a per-chat file whose open - // failed before the listing. + // The listed chats the tab list left to it: no current row, corrupt, unreadable, or still in a + // per-chat file. restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), ...startup } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts index 96d914bd9c87..b10df671fc43 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts @@ -10,11 +10,11 @@ import { updateTestJournalRowJson } from '../agent-session-journal/journal-host-database-test-support' import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' +import { createRestTestRig, type RestTestRig } from './structured-agent-session-rest-test-rig' import { - createRestTestRig, - restTestChat, - type RestTestRig -} from './structured-agent-session-rest-test-rig' + crashRestTestChatMidTurn, + runRestTestStartup +} from './structured-agent-session-rest-test-startup' const SESSION = 'session-damaged' const rigs: RestTestRig[] = [] @@ -27,17 +27,6 @@ afterEach(async () => { vi.restoreAllMocks() }) -/** What startup runs, in order, on the ids the tab list names. */ -async function startup(rig: RestTestRig): Promise { - const listed = rig.store.getVisibleSessionTabIndex().sessionIds - await rig.host.reconcileRestartLeases() - await rig.host.startup.catchUpMissingStatuses(listed) - await rig.host.startup.restoreListedFromPerChatFiles(listed) - const background = rig.host.startup.seedStoredStatuses(listed) - await rig.host.startup.settleOwedSessions(listed) - await rig.host.restoreReadableSessions(background) -} - /** Each warning startup logged about the chat. */ function warningsFor(warn: ReturnType): unknown[] { return warn.mock.calls.filter( @@ -54,16 +43,7 @@ it.each([true, false])( async (listed) => { const rig = await createRestTestRig() rigs.push(rig) - await restTestChat(rig, SESSION, { message: 'asked', listed }) - const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value - await rig.host - .collaboratorsForTests() - .sessions.get(SESSION)! - .journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { fence: rig.store.getRecord(SESSION)!.lease.runtimeFence, turnScope: { kind: 'thread' } } - ) + await crashRestTestChatMidTurn(rig, SESSION, { listed }) await rig.crash() const { db } = openTestJournalHostDatabase(rig.root) // Damaged in place: the row still names the chat's tip, so startup selects it. @@ -73,7 +53,7 @@ it.each([true, false])( await rig.boot() const warn = vi.spyOn(rig.host.deps.logger, 'warn') - await startup(rig) + await runRestTestStartup(rig) expect(warningsFor(warn)).toHaveLength(1) expect(readTestJournalSessionStatus(rig.root, SESSION)).toBeNull() @@ -84,7 +64,7 @@ it.each([true, false])( await rig.crash() await rig.boot() const again = vi.spyOn(rig.host.deps.logger, 'warn') - await startup(rig) + await runRestTestStartup(rig) expect(rig.journalOpens.mock.calls.filter(([id]) => id === SESSION)).toEqual( listed ? [[SESSION]] : [] ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.test.ts deleted file mode 100644 index f396651d74bc..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-lease-phase.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -// Startup's one lease phase: the tab listing waits only on the recoveries of the chats opened before -// it, and no lease is checked again or recovered twice, whichever restore asks for it. - -import { afterEach, describe, expect, it, vi } from 'vitest' -import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' -import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' -import { - createRestTestRig, - restTestChat, - type RestTestRig -} from './structured-agent-session-rest-test-rig' -import { latestRestTestStatus } from './structured-agent-session-rest-test-observations' -import { moveRestTestChatToPerChatFile } from './structured-agent-session-rest-test-per-chat-file' - -const rigs: RestTestRig[] = [] - -afterEach(async () => { - for (const rig of rigs.splice(0)) { - await rig.dispose() - } - closeTestJournalHostDatabases() - vi.restoreAllMocks() -}) - -const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds - -/** - * A chat whose provider process was up when Orca died, left in a per-chat file when `inFile`. Its - * process check answers once, which leaves its lease `recovering`, then never again; answers how - * many checks hung. - */ -async function crashWithHungRecovery( - rig: RestTestRig, - sessionId: string, - { listed, inFile }: { listed: boolean; inFile: boolean } -): Promise<() => number> { - rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) - await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) - // Held mid-send, before anything releases it. - const lease = rig.store.getRecord(sessionId)!.lease - await rig.host.flushAllStreamedEvents() - await rig.crash() - await writeOlderBuildLease(rig.root, sessionId, { ...lease }) - if (inFile) { - await moveRestTestChatToPerChatFile(rig, sessionId) - } - let answered = false - let hung = 0 - rig.probeOwner.mockImplementation(async (record) => { - if (record.sessionId !== sessionId) { - return { outcome: 'pid-absent' } - } - if (!answered) { - answered = true - return { outcome: 'identity-matched', matchedOn: ['spawn-token'] } - } - hung += 1 - return new Promise(() => {}) - }) - return () => hung -} - -/** Startup up to the listing's answer, as the host's step runs it; answers the restore's list. */ -async function startupThroughListing(rig: RestTestRig): Promise { - const listed = listedIds(rig) - await rig.host.reconcileRestartLeases() - await rig.host.startup.catchUpMissingStatuses(listed) - await rig.host.startup.restoreListedFromPerChatFiles(listed) - return rig.host.startup.seedStoredStatuses(listed) -} - -const within = (work: Promise, ms: number) => - Promise.race([ - work.then(() => 'done'), - new Promise((resolve) => setTimeout(() => resolve('still waiting'), ms)) - ]) - -describe('startup lease recovery, per lease', () => { - it("answers the listing without waiting on a tabless chat's hung recovery", async () => { - const rig = await createRestTestRig() - rigs.push(rig) - await restTestChat(rig, 'session-file', { message: 'asked session-file' }) - const hung = await crashWithHungRecovery(rig, 'session-tabless', { - listed: false, - inFile: false - }) - await moveRestTestChatToPerChatFile(rig, 'session-file') - vi.spyOn(rig.host.deps.logger, 'warn') - await rig.boot({ startupRecoveryBudgetMs: 10_000 }) - expect(listedIds(rig)).toEqual(['session-file']) - - // The listed per-chat-file chat is opened before the listing; its own lease is settled. - expect(await within(startupThroughListing(rig), 5_000)).toBe('done') - expect(rig.host.hasSession('session-file')).toBe(true) - expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ status: 'idle' }) - // The tabless chat's recovery is still running, started before the listing; the settle waits. - expect(hung()).toBe(1) - await rig.host.startup.settleOwedSessions(listedIds(rig)) - expect(hung()).toBe(1) - }, 30_000) - - it('does not check a listed per-chat-file chat again after the listing once its recovery outlasted startup', async () => { - const rig = await createRestTestRig() - rigs.push(rig) - const hung = await crashWithHungRecovery(rig, 'session-file', { listed: true, inFile: true }) - vi.spyOn(rig.host.deps.logger, 'warn') - await rig.boot({ startupRecoveryBudgetMs: 50 }) - - expect(await within(startupThroughListing(rig), 5_000)).toBe('done') - // Opened unverified before the listing: the hung check is left to the next attach or send. - expect(rig.host.hasSession('session-file')).toBe(true) - expect(rig.store.getRecord('session-file')!.lease.handoffStage).toBe('recovering') - // The restore after the listing, given every listed chat, as when startup gave it no list. - expect(await within(rig.host.restoreReadableSessions(listedIds(rig)), 3_000)).toBe('done') - expect(hung()).toBe(1) - }, 30_000) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts deleted file mode 100644 index c8af43340fa6..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-per-chat-file-restore.ts +++ /dev/null @@ -1,51 +0,0 @@ -// Listed chats whose history is still in an older build's per-chat file have no stored status to -// list them from: the restart restore's own per-chat worker opens each one from its file before -// the tab listing answers, settles what a gone process left and publishes its status, and the -// restore after the listing skips it. A chat whose open fails costs only itself and stays listed. - -import { readJournalSessionStatuses } from '../agent-session-journal/journal-session-state' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { hasHistoryOutsideJournalDatabase } from './structured-agent-session-read-restore' -import type { - StructuredAgentSessionStartupLeases, - StructuredAgentSessionStartupStateDeps -} from './structured-agent-session-startup-state' - -// The listing waits for the whole pass, so a few at once: one slow file or recovery holds one lane. -const PRE_LISTING_RESTORE_CONCURRENCY = 4 - -/** Never rejects. `leases` resolves each chat's own lease only, so the listing waits on no other - * chat's recovery. */ -export async function restoreListedFromPerChatFiles( - deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[], - leases: StructuredAgentSessionStartupLeases -): Promise { - const database = deps.openDeps.journalDatabase - // A newer build's database: the restore after the listing reads every listed chat as before. - if (database.readOnly || deps.isDisposed()) { - return - } - try { - const inDatabase = new Set( - readJournalSessionStatuses(database.db, listedIds).map(({ sessionId }) => sessionId) - ) - const records = listedIds.flatMap((sessionId): AgentSessionRecord[] => { - const record = inDatabase.has(sessionId) ? null : deps.openDeps.store.getRecord(sessionId) - return record && - deps.canSettle(record) && - !deps.hasSession(sessionId) && - hasHistoryOutsideJournalDatabase(database, record) - ? [record] - : [] - }) - if (records.length > 0) { - await deps.restoreListed(records, leases, PRE_LISTING_RESTORE_CONCURRENCY) - } - } catch (error) { - deps.openDeps.logger.warn('restoring chats still in per-chat files at startup failed', { - scope: 'startup-per-chat-file-restore', - error - }) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index f175745ea3a9..ed92fc7bf4f7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -16,11 +16,6 @@ import { } from '../agent-session-journal/journal-host-database-test-support' import Database from '../../sqlite/sync-database' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' -import { - createStructuredAgentSessionStartupState, - type StructuredAgentSessionStartupStateDeps -} from './structured-agent-session-startup-state' import { readUnsettledJournalSessionIds } from '../agent-session-journal/journal-session-state' import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' import { writeOlderBuildLease } from '../../runtime/agent-session-older-build-lease.test-fixture' @@ -33,11 +28,14 @@ import { sendRestTestMessage, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { + crashRestTestChatMidTurn, + runRestTestStartup +} from './structured-agent-session-rest-test-startup' import { latestRestTestStatus, restTestOpens } from './structured-agent-session-rest-test-observations' -import { moveRestTestChatToPerChatFile } from './structured-agent-session-rest-test-per-chat-file' const rigs: RestTestRig[] = [] @@ -65,33 +63,6 @@ async function crashMidSend(rig: RestTestRig, sessionId: string, listed = true): await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) } -/** A chat whose turn is running when Orca dies: startup selects it to settle. */ -async function crashMidTurn(rig: RestTestRig, sessionId: string): Promise { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - const [{ providerIdentity }] = await Promise.all( - rig.adapter.dispatch.mock.results.slice(-1).map((result) => result.value) - ) - await rig.host - .collaboratorsForTests() - .sessions.get(sessionId)! - .journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } - ) -} - -/** What startup runs, in order, on the ids the tab list names. */ -async function startup(rig: RestTestRig, listed: readonly string[]) { - await rig.host.reconcileRestartLeases() - await rig.host.startup.catchUpMissingStatuses(listed) - await rig.host.startup.restoreListedFromPerChatFiles(listed) - const background = rig.host.startup.seedStoredStatuses(listed) - await rig.host.startup.settleOwedSessions(listed) - await rig.host.restoreReadableSessions(background) - return background -} - function opened(rig: RestTestRig, ids: readonly string[]): string[] { return ids.filter((sessionId) => restTestOpens(rig, sessionId) > 0) } @@ -105,103 +76,6 @@ function statusRows(rig: RestTestRig, sessionId: string): AgentSessionStatusSumm const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds describe('seeding statuses from stored state', () => { - it('seeds each settled chat exactly as its open would publish it (T5)', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-answered', { message: 'finished work' }) - await restTestChat(rig, 'session-quiet') - rig.adapter.dispatch.mockResolvedValueOnce({ - state: 'rejected', - ...agentSessionFailureWords(agentSessionFailureFact('providerRejected'), { - surface: 'rejection' - }) - }) - await restTestChat(rig, 'session-refused', { message: 'refused' }) - await rig.host.flushAllStreamedEvents() - const ids = ['session-answered', 'session-quiet', 'session-refused'] - closeTestJournalHostDatabases() - const readRoot = `${rig.root}-read` - await cp(rig.root, readRoot, { recursive: true }) - const readRig = await newRig(readRoot) - - const host = await rig.boot() - await host.reconcileRestartLeases() - expect(host.startup.seedStoredStatuses(ids)).toEqual([]) - expect(opened(rig, ids)).toEqual([]) - // The same chats in another run, each opened by a read. - await readRig.host.reconcileRestartLeases() - for (const sessionId of ids) { - await readRig.host.history({ sessionId, direction: 'tail' }) - } - - for (const sessionId of ids) { - // Every field, `updatedAt` included: both come from the journal's own newest activity. - expect(latestRestTestStatus(rig, sessionId)).toEqual(latestRestTestStatus(readRig, sessionId)) - } - expect(latestRestTestStatus(rig, 'session-refused')).toMatchObject({ turnOutcome: 'failure' }) - // Opening a seeded chat finds its row equal and sends nothing. - const seeded = ids.map((sessionId) => statusRows(rig, sessionId).length) - for (const sessionId of ids) { - await host.history({ sessionId, direction: 'tail' }) - } - expect(ids.map((sessionId) => statusRows(rig, sessionId).length)).toEqual(seeded) - }) - - it('seeds a crash-cut turn with the verdict its open publishes (T5, interrupted and unconfirmed)', async () => { - const rig = await newRig() - const ids = ['session-interrupted', 'session-unconfirmed'] - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - const [{ providerIdentity }] = await Promise.all( - rig.adapter.dispatch.mock.results.slice(-1).map((result) => result.value) - ) - const session = rig.host.collaboratorsForTests().sessions.get(sessionId)! - // The turn's own row, beside the accepted message the dispatch's identity names. - await session.journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { - fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, - turnScope: { kind: 'thread' } - } - ) - } - await rig.crash() - // The unconfirmed chat's owner can never be judged; the other's is proven gone. - const probe = async (record: { sessionId: string }) => - record.sessionId === 'session-unconfirmed' - ? { outcome: 'indeterminate' as const, reason: 'no start time' } - : { outcome: 'pid-absent' as const } - rig.probeOwner.mockImplementation(probe) - await rig.boot() - await startup(rig, ids) - await rig.host.flushAllStreamedEvents() - await rig.crash() - closeTestJournalHostDatabases() - const readRoot = `${rig.root}-read` - await cp(rig.root, readRoot, { recursive: true }) - const readRig = await newRig(readRoot) - readRig.probeOwner.mockImplementation(probe) - - const host = await rig.boot() - await host.reconcileRestartLeases() - expect(host.startup.seedStoredStatuses(ids)).toEqual([]) - expect(opened(rig, ids)).toEqual([]) - await readRig.host.reconcileRestartLeases() - for (const sessionId of ids) { - await readRig.host.history({ sessionId, direction: 'tail' }) - } - - for (const sessionId of ids) { - expect(latestRestTestStatus(rig, sessionId)).toEqual(latestRestTestStatus(readRig, sessionId)) - } - expect(latestRestTestStatus(rig, 'session-interrupted')).toMatchObject({ - turnOutcome: 'interruption' - }) - expect(latestRestTestStatus(rig, 'session-unconfirmed')).toMatchObject({ - turnOutcome: 'unconfirmed' - }) - }) - it('has every settled chat in the first snapshot a later subscriber gets (T8, T17)', async () => { const rig = await newRig() for (const sessionId of ['session-1', 'session-2']) { @@ -275,6 +149,136 @@ describe('seeding statuses from stored state', () => { }) }) +/** A copy of the rig's files for a second host, which reads each chat directly; taken while + * nothing is writing. */ +async function readCopy( + rig: RestTestRig, + probe?: Parameters[0] +): Promise { + closeTestJournalHostDatabases() + const readRoot = `${rig.root}-read` + await cp(rig.root, readRoot, { recursive: true }) + const readRig = await newRig(readRoot) + if (probe) { + readRig.probeOwner.mockImplementation(probe) + } + return readRig +} + +async function readEach(rig: RestTestRig, ids: readonly string[]): Promise { + await rig.host.reconcileRestartLeases() + for (const sessionId of ids) { + await rig.host.history({ sessionId, direction: 'tail' }) + } +} + +describe('startup statuses end where a direct read of each chat lands', () => { + it('seeds each settled chat exactly as its open would publish it, crash-cut verdicts included (T5)', async () => { + const rig = await newRig() + await restTestChat(rig, 'session-answered', { message: 'finished work' }) + await restTestChat(rig, 'session-quiet') + rig.adapter.dispatch.mockResolvedValueOnce({ + state: 'rejected', + ...agentSessionFailureWords(agentSessionFailureFact('providerRejected'), { + surface: 'rejection' + }) + }) + await restTestChat(rig, 'session-refused', { message: 'refused' }) + for (const sessionId of ['session-interrupted', 'session-unconfirmed']) { + await crashRestTestChatMidTurn(rig, sessionId) + } + const ids = [ + 'session-answered', + 'session-quiet', + 'session-refused', + 'session-interrupted', + 'session-unconfirmed' + ] + await rig.crash() + // The unconfirmed chat's owner can never be judged; every other one is proven gone. + const probe = async (record: { sessionId: string }) => + record.sessionId === 'session-unconfirmed' + ? { outcome: 'indeterminate' as const, reason: 'no start time' } + : { outcome: 'pid-absent' as const } + rig.probeOwner.mockImplementation(probe) + // The first launch settles the crash-cut turns; the next seeds every chat from its row. + await rig.boot() + await runRestTestStartup(rig, ids) + await rig.host.flushAllStreamedEvents() + await rig.crash() + const readRig = await readCopy(rig, probe) + + const host = await rig.boot() + await host.reconcileRestartLeases() + expect(host.startup.seedStoredStatuses(ids)).toEqual([]) + expect(opened(rig, ids)).toEqual([]) + await readEach(readRig, ids) + + for (const sessionId of ids) { + // Every field, `updatedAt` included: both come from the journal's own newest activity. + expect(latestRestTestStatus(rig, sessionId)).toEqual(latestRestTestStatus(readRig, sessionId)) + } + expect(latestRestTestStatus(rig, 'session-refused')).toMatchObject({ turnOutcome: 'failure' }) + expect(latestRestTestStatus(rig, 'session-interrupted')).toMatchObject({ + turnOutcome: 'interruption' + }) + expect(latestRestTestStatus(rig, 'session-unconfirmed')).toMatchObject({ + turnOutcome: 'unconfirmed' + }) + // Opening a seeded chat finds its row equal and sends nothing. + const seeded = ids.map((sessionId) => statusRows(rig, sessionId).length) + for (const sessionId of ids) { + await host.history({ sessionId, direction: 'tail' }) + } + expect(ids.map((sessionId) => statusRows(rig, sessionId).length)).toEqual(seeded) + }) + + it('ends every chat the restore after the listing opens where a direct read lands, with no pre-crash work (T8)', async () => { + const rig = await newRig() + const ids = ['session-settled', 'session-quiet', 'session-crashed', 'session-pending'] + // Two a clean quit settled, one cut off mid-turn by a crash, and one whose send the provider + // only admitted, left pending below a fence that has since moved. + await restTestChat(rig, 'session-settled', { message: 'finished work' }) + await restTestChat(rig, 'session-quiet') + await rig.host.flushAllStreamedEvents() + await rig.boot() + await restTestChat(rig, 'session-crashed', { message: 'cut off' }) + rig.adapter.dispatch.mockResolvedValueOnce({ state: 'admitted' }) + await restTestChat(rig, 'session-pending', { message: 'only admitted' }) + await rig.crash() + // A live lease names the link proven at its fence, so the fence moves with that link. + await rig.store.transitionHandoff('session-pending', (record) => ({ + ...record, + lease: { ...record.lease, runtimeFence: record.lease.runtimeFence + 1 }, + providerHandleChain: record.providerHandleChain.map((link, index, chain) => + index === chain.length - 1 ? { ...link, mintedAtFence: link.mintedAtFence + 1 } : link + ) + })) + const readRig = await readCopy(rig) + + const host = await rig.boot() + await host.reconcileRestartLeases() + await host.restoreReadableSessions(ids) + await readEach(readRig, ids) + + // `updatedAt` is when the row was projected, a wall-clock read that differs between two boots. + const rows = (of: RestTestRig): Record> => + Object.fromEntries( + ids.flatMap((sessionId) => { + const summary = latestRestTestStatus(of, sessionId) + if (!summary) { + return [] + } + const { updatedAt: _updatedAt, ...row } = summary + return [[sessionId, row]] + }) + ) + await vi.waitFor(() => expect(rows(rig)).toEqual(rows(readRig))) + expect(Object.keys(rows(rig))).toEqual(ids) + expect(statusRows(rig, 'session-crashed').map((row) => row.status)).not.toContain('working') + }) +}) + describe('startup opens only what it must (T6, T7, T14)', () => { it('settles crashed chats with or without a tab, and opens only what stored state cannot answer', async () => { const rig = await newRig() @@ -315,16 +319,14 @@ describe('startup opens only what it must (T6, T7, T14)', () => { await rig.boot() const listed = listedIds(rig) expect(listed).not.toContain('session-crashed-closed') - const background = await startup(rig, listed) + const background = await runRestTestStartup(rig, listed) const unlisted = ['session-crashed-closed'] - // The rowless chat's row comes from its rows alone: it is never opened. expect(opened(rig, [...listed, ...unlisted]).toSorted()).toEqual( - ['session-crashed', ...unlisted, 'session-uncopied'].toSorted() + ['session-crashed', ...unlisted, 'session-rowless', 'session-uncopied'].toSorted() ) - // Its row is derived before the seed, and the per-chat file is opened before it: nothing is - // left to the restore after the listing. - expect(background).toEqual([]) + // No stored status to answer from: both are left to the restore after the listing. + expect(background).toEqual(['session-rowless', 'session-uncopied']) expect(latestRestTestStatus(rig, 'session-draft')).toMatchObject({ status: 'idle' }) // Settled with no user action; the listed one is open and never showed its pre-crash work. for (const sessionId of ['session-crashed', ...unlisted]) { @@ -347,7 +349,7 @@ describe('startup opens only what it must (T6, T7, T14)', () => { expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) } - // The missing status was written back before the seed, and its row published, with no open. + // The rowless chat's open writes its status back and publishes it. expect(readTestJournalSessionStatus(rig.root, 'session-rowless')).toMatchObject({ lifecycle: 'idle' }) @@ -356,7 +358,7 @@ describe('startup opens only what it must (T6, T7, T14)', () => { // The next boot finds nothing to settle and nothing without a status (T4b). await rig.crash() await rig.boot() - const again = await startup(rig, listedIds(rig)) + const again = await runRestTestStartup(rig) expect(again).toEqual([]) expect(opened(rig, [...listedIds(rig), ...unlisted])).toEqual([]) }) @@ -393,7 +395,7 @@ describe('a provider process that outlived the crash (R1T-2)', () => { ) await rig.boot({ stopOwnerProcess }) - await startup(rig, listedIds(rig)) + await runRestTestStartup(rig) expect(stopOwnerProcess).toHaveBeenCalled() expect(alive).toBe(false) @@ -410,165 +412,97 @@ describe('a provider process that outlived the crash (R1T-2)', () => { await rig.crash() stopOwnerProcess.mockClear() await rig.boot({ stopOwnerProcess }) - await startup(rig, listedIds(rig)) + await runRestTestStartup(rig) expect(stopOwnerProcess).not.toHaveBeenCalled() }) - it('is stopped for an unlisted chat whose lease a command checks after the lease check failed', async () => { - const rig = await newRig() - const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-listed', 'session-closed']) - const listed = listedIds(rig) - const failed = new Error('disk I/O error') - vi.spyOn(rig.store, 'reconcileOnRestart') - .mockRejectedValueOnce(failed) - .mockRejectedValueOnce(failed) - - await rig.host.reconcileRestartLeases() - expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) - // An attach let through mid-catch-up checks the leases first, moving them to `recovering`; - // what it answers after that does not matter here. - const catchingUp = rig.host.startup.catchUpMissingStatuses(listed) - await restTestChat(rig, 'session-listed').catch(() => undefined) - expect(rig.store.getRecord('session-closed')!.lease.handoffStage).toBe('recovering') - await catchingUp - await rig.host.startup.restoreListedFromPerChatFiles(listed) - rig.host.startup.seedStoredStatuses(listed) - // Owed: the crash cut its turn, so the settle opens it. - expect(readUnsettledJournalSessionIds(db(rig))).toContain('session-closed') - await rig.host.startup.settleOwedSessions(listed) - - expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) - }) - - it("is stopped for an unlisted chat when the lease check's first try fails", async () => { - const rig = await newRig() - const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-open', 'session-closed']) - const listed = listedIds(rig) - expect(listed).toEqual(['session-open']) - const reconcile = vi - .spyOn(rig.store, 'reconcileOnRestart') - .mockRejectedValueOnce(new Error('disk I/O error')) - - await startup(rig, listed) - - // The lease check's second try settles every lease; no restore checks them again. - expect(reconcile).toHaveBeenCalledTimes(2) - expect(stopOwnerProcess).toHaveBeenCalledTimes(1) - expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) - }) - - it('is stopped for an unlisted chat whose lease a command checks once the settle is running', async () => { - const rig = await newRig() - const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-listed', 'session-closed']) - const listed = listedIds(rig) - const failed = new Error('disk I/O error') - const reconcile = vi - .spyOn(rig.store, 'reconcileOnRestart') - .mockRejectedValueOnce(failed) - .mockRejectedValueOnce(failed) - await rig.host.reconcileRestartLeases() - await rig.host.startup.catchUpMissingStatuses(listed) - await rig.host.startup.restoreListedFromPerChatFiles(listed) - rig.host.startup.seedStoredStatuses(listed) - - const settling = rig.host.startup.settleOwedSessions(listed) - // The settle has waited on its recoveries and read its records; an attach the gate let through - // now checks the leases, and what it answers after that does not matter here. - await new Promise((resolve) => setImmediate(resolve)) - expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) - await restTestChat(rig, 'session-listed').catch(() => undefined) - await settling - - expect(reconcile).toHaveBeenCalledTimes(3) - expect(stopOwnerProcess).toHaveBeenCalledTimes(1) - expectInterruptedAndStopped(rig, 'session-closed', stopOwnerProcess) - }) - - it('is stopped for a listed chat still in its per-chat file whose lease a command checks after the lease check failed', async () => { - const rig = await newRig() - const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-file', 'session-other'], { - live: 'session-file', - beforeBoot: () => moveRestTestChatToPerChatFile(rig, 'session-file') - }) - const listed = listedIds(rig) - expect(listed).toEqual(['session-file']) - const failed = new Error('disk I/O error') - vi.spyOn(rig.store, 'reconcileOnRestart') - .mockRejectedValueOnce(failed) - .mockRejectedValueOnce(failed) - await rig.host.reconcileRestartLeases() - // An attach let through before the listing checks the leases; its own answer does not matter. - await restTestChat(rig, 'session-other', { listed: false }).catch(() => undefined) - expect(rig.store.getRecord('session-file')!.lease.handoffStage).toBe('recovering') - - await rig.host.startup.catchUpMissingStatuses(listed) - await rig.host.startup.restoreListedFromPerChatFiles(listed) - - // Opened before the listing from its file, after its recovery: the turn reads as interrupted. - expect(rig.host.hasSession('session-file')).toBe(true) - expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ turnOutcome: 'interruption' }) - expect(stopOwnerProcess).toHaveBeenCalledTimes(1) - }) - - it('leaves every lease to the next attach or send when the lease check fails twice', async () => { - const rig = await newRig() - const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-open', 'session-closed']) - const listed = listedIds(rig) - const failed = new Error('disk I/O error') - const reconcile = vi - .spyOn(rig.store, 'reconcileOnRestart') - .mockRejectedValueOnce(failed) - .mockRejectedValueOnce(failed) - - await startup(rig, listed) - - // No startup restore checks the leases again: each chat settles unverified, nothing stopped. - expect(reconcile).toHaveBeenCalledTimes(2) - expect(stopOwnerProcess).not.toHaveBeenCalled() - expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) - expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ - lifecycle: 'idle', - summary: { turnOutcome: 'unconfirmed' } - }) - }) + it.each([ + { check: "'s first try fails", fails: 1, command: null, checks: 2, stopped: 1 }, + { check: ' fails twice', fails: 2, command: null, checks: 2, stopped: 0 }, + { + check: ' fails twice and a command checks the leases before the settle', + fails: 2, + command: 'before the settle', + checks: 3, + stopped: 1 + }, + { + check: ' fails twice and a command checks the leases once the settle is running', + fails: 2, + command: 'once the settle is running', + checks: 3, + stopped: 1 + } + ] as const)( + 'settles an unlisted chat from what its recovery found when the lease check$check', + async ({ fails, command, checks, stopped }) => { + const rig = await newRig() + const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-listed', 'session-closed']) + const listed = listedIds(rig) + expect(listed).toEqual(['session-listed']) + const reconcile = vi.spyOn(rig.store, 'reconcileOnRestart') + for (let failure = 0; failure < fails; failure += 1) { + reconcile.mockRejectedValueOnce(new Error('disk I/O error')) + } + // An attach the gate let through checks the leases first; what it answers does not matter. + const attach = () => restTestChat(rig, 'session-listed').catch(() => undefined) + + await rig.host.reconcileRestartLeases() + if (command === 'before the settle') { + expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) + await attach() + expect(rig.store.getRecord('session-closed')!.lease.handoffStage).toBe('recovering') + } + const background = rig.host.startup.seedStoredStatuses(listed) + // Owed: the crash cut its turn, so the settle opens it. + expect(readUnsettledJournalSessionIds(db(rig))).toContain('session-closed') + const settling = rig.host.startup.settleOwedSessions(listed) + if (command === 'once the settle is running') { + // The settle has waited on its recoveries and read its records. + await new Promise((resolve) => setImmediate(resolve)) + expect(rig.store.getRecord('session-closed')!.lease.unreconciled).toBe(true) + await attach() + } + await settling + await rig.host.restoreReadableSessions(background) + + // No startup restore checks the leases again; with no check that held, nothing is stopped + // and the chat settles unverified. + expect(reconcile).toHaveBeenCalledTimes(checks) + expect(stopOwnerProcess).toHaveBeenCalledTimes(stopped) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle', + summary: { turnOutcome: stopped ? 'interruption' : 'unconfirmed' } + }) + expect(rig.store.getRecord('session-closed')!.lease).toMatchObject( + stopped ? { deathEvidence: { kind: 'pid-absent' } } : { unreconciled: true } + ) + expect(rig.host.hasSession('session-closed')).toBe(false) + } + ) }) -/** - * Chats whose turn was running when Orca died. The first is listed, the rest have no tab; the `live` - * one's provider process (the last, unless named) is still up until a stop ends it. `beforeBoot` - * runs between the crash and the next launch. - */ +/** Chats whose turn was running when Orca died; the first is listed. The last one's provider + * process is still up until a stop ends it. */ async function crashWithLiveOwner( rig: RestTestRig, - ids: readonly string[], - options: { live?: string; beforeBoot?: () => Promise } = {} + ids: readonly string[] ): Promise> { const leases: AgentSessionRecord['lease'][] = [] for (const [index, sessionId] of ids.entries()) { - const listed = index === 0 && ids.length > 1 - await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) - const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value - await rig.host - .collaboratorsForTests() - .sessions.get(sessionId)! - .journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } - ) + await crashRestTestChatMidTurn(rig, sessionId, { listed: index === 0 }) leases.push(rig.store.getRecord(sessionId)!.lease) } await rig.crash() for (const lease of leases) { await writeOlderBuildLease(rig.root, lease.sessionId, { ...lease }) } - await options.beforeBoot?.() let alive = true const stopOwnerProcess = vi.fn(() => { alive = false }) rig.probeOwner.mockImplementation(async (record) => - alive && record.sessionId === (options.live ?? ids.at(-1)) + alive && record.sessionId === ids.at(-1) ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } ) @@ -576,24 +510,6 @@ async function crashWithLiveOwner( return stopOwnerProcess } -/** Settled from the death evidence its recovery recorded: the turn reads as interrupted, not as - * unconfirmed, and the chat is closed again. */ -function expectInterruptedAndStopped( - rig: RestTestRig, - sessionId: string, - stopOwnerProcess: ReturnType -): void { - expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ - lifecycle: 'idle', - summary: { turnOutcome: 'interruption' } - }) - expect(stopOwnerProcess).toHaveBeenCalled() - expect(rig.store.getRecord(sessionId)!.lease).toMatchObject({ - deathEvidence: { kind: 'pid-absent' } - }) - expect(rig.host.hasSession(sessionId)).toBe(false) -} - describe('one awaited settle covers a chat whose tab closes meanwhile (R1T-4)', () => { it('settles a listed chat the listed worker skips because its tab closed', async () => { const rig = await newRig() @@ -617,62 +533,14 @@ describe('one awaited settle covers a chat whose tab closes meanwhile (R1T-4)', }) }) -describe('the startup gate holds chat commands and never refuses them', () => { - async function bootGated(gate: StructuredAgentSessionStartupGate): Promise { - const rig = await newRig() - await restTestChat(rig, 'session-1', { message: 'first' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - gate.hold() - await rig.boot({ commandsReady: gate.ready }) - return rig - } - - it('holds a send until the settle ends, then delivers it; listing and status answer meanwhile', async () => { - const gate = new StructuredAgentSessionStartupGate() - const rig = await bootGated(gate) - // The tab list and the seeded status answer while the gate is closed. - expect(listedIds(rig)).toContain('session-1') - rig.host.startup.seedStoredStatuses(['session-1']) - expect(latestRestTestStatus(rig, 'session-1')).toMatchObject({ status: 'idle' }) - const dispatched = rig.adapter.dispatch.mock.calls.length - let answered = false - const sent = sendRestTestMessage(rig, 'session-1', 'during startup').then((result) => { - answered = true - return result - }) - await new Promise((resolve) => setTimeout(resolve, 50)) - expect(answered).toBe(false) - expect(rig.adapter.dispatch.mock.calls.length).toBe(dispatched) - - gate.openWhen(Promise.reject(new Error('settle failed'))) - - expect(await sent).toMatchObject({ ok: true }) - await vi.waitFor(() => - expect(rig.adapter.dispatch.mock.calls.length).toBeGreaterThan(dispatched) - ) - }) - - it('lets held commands through at its ceiling if the settle never ends', async () => { - const gate = new StructuredAgentSessionStartupGate(30) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const rig = await bootGated(gate) - - expect(await sendRestTestMessage(rig, 'session-1', 'after the ceiling')).toMatchObject({ - ok: true - }) - expect(gate.ready()).toBeNull() - }) -}) - describe('commands held for the real startup settle never deadlock it (R2T-1)', () => { const CEILING_MS = 3_000 - it('lets a command issued before the settle, one during it and a healthy read through as the settle ends', async () => { + it('holds a command issued before the settle, lets it, one during it and a healthy read through as the settle ends', async () => { vi.spyOn(console, 'warn').mockImplementation(() => undefined) const rig = await newRig() for (const sessionId of ['session-a', 'session-b']) { - await crashMidTurn(rig, sessionId) + await crashRestTestChatMidTurn(rig, sessionId) } await restTestChat(rig, 'session-healthy', { message: 'fine' }) await rig.crash() @@ -685,10 +553,22 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', expect(readUnsettledJournalSessionIds(db(rig)).toSorted()).toEqual(['session-a', 'session-b']) const started = Date.now() const elapsed = () => Date.now() - started + const dispatched = rig.adapter.dispatch.mock.calls.length // Before the settle: a send to a crashed chat, and the option read a chat pane fires on mount. - const sendBefore = sendRestTestMessage(rig, 'session-a', 'during startup').then(elapsed) + let sendAnswered = false + const sendBefore = sendRestTestMessage(rig, 'session-a', 'during startup').then((result) => { + sendAnswered = true + expect(result).toMatchObject({ ok: true }) + return elapsed() + }) const optionsBefore = rig.host.readOptions('session-a').then(elapsed) + await new Promise((resolve) => setTimeout(resolve, 20)) + // Held, neither answered nor dispatched; the tab list and the seeded status answer meanwhile. + expect(sendAnswered).toBe(false) + expect(rig.adapter.dispatch.mock.calls.length).toBe(dispatched) + expect(listedIds(rig)).toContain('session-healthy') + expect(latestRestTestStatus(rig, 'session-healthy')).toMatchObject({ status: 'idle' }) const settled = rig.host.startup.settleOwedSessions(listed) gate.openWhen(settled) // During the settle: the second crashed chat, which the settle has not reached yet. @@ -702,6 +582,9 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', expect(at).toBeLessThan(CEILING_MS / 2) } expect(gate.ready()).toBeNull() + await vi.waitFor(() => + expect(rig.adapter.dispatch.mock.calls.length).toBeGreaterThan(dispatched) + ) }, 20_000) it('holds a rewind and a /clear sent straight to the host, and a read, then answers them as the settle ends', async () => { @@ -709,7 +592,7 @@ describe('commands held for the real startup settle never deadlock it (R2T-1)', const rig = await newRig() const crashed = ['session-a', 'session-b', 'session-c'] for (const sessionId of crashed) { - await crashMidTurn(rig, sessionId) + await crashRestTestChatMidTurn(rig, sessionId) } await rig.crash() const gate = new StructuredAgentSessionStartupGate(CEILING_MS) @@ -779,86 +662,67 @@ function conversationCommandParams(sessionId: string, command: 'clear') { } } -describe('a recovery that never answers', () => { - it('leaves that lease unverified and lets the settle go on', async () => { - const logger = { warn: vi.fn(), error: vi.fn() } - const stuck = { ...agentSessionRecordFixture(), sessionId: 'session-stuck' } - stuck.lease = { ...stuck.lease, sessionId: 'session-stuck', handoffStage: 'recovering' } - const resolveRecovery = vi.fn(() => new Promise(() => {})) - const restoreListed = vi.fn(async () => undefined) - const openDeps = { - store: { getRecord: () => stuck, listRecords: () => [stuck] }, - journalDatabase: { readOnly: false, db: { prepare: () => ({ all: () => [] }) } }, - logger - } - const state = createStructuredAgentSessionStartupState({ - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the settle reads only the store's records and the unsettled-row query; nothing is selected, so nothing opens. - openDeps: openDeps as unknown as StructuredAgentSessionStartupStateDeps['openDeps'], - canSettle: (record): record is AgentSessionRecord => record !== null, - seedStatus: vi.fn(), - reconcile: async () => true, - resolveRecovery, - restoreListed, - serialize: (_sessionId, task) => task(), - hasSession: () => false, - isListed: () => true, - isDisposed: () => false, - recoveryBudgetMs: 20 - }) - - await state.settleOwedSessions([]) - - expect(resolveRecovery).toHaveBeenCalledWith('session-stuck') - expect(restoreListed).toHaveBeenCalledWith([], { - reconcile: expect.any(Function), - resolveRecovery: expect.any(Function) - }) - expect(stuck.lease.handoffStage).toBe('recovering') - expect(logger.warn).toHaveBeenCalledWith('a chat recovery outlasted startup; left unverified', { - scope: 'startup-recovery-timeout', - sessionId: 'session-stuck' - }) - }) -}) - describe('a listed chat whose recovery never answers', () => { - it('is opened unverified, with no second recovery beside the first, and the rest still settle', async () => { - const rig = await newRig() - await crashMidSend(rig, 'session-stuck') - await crashMidSend(rig, 'session-closed', false) - const lease = rig.store.getRecord('session-stuck')!.lease - await rig.crash() - // Its provider process was up when Orca died; the check of it answers once, then never again. - await writeOlderBuildLease(rig.root, 'session-stuck', { ...lease }) - let answered = false - let hung = 0 - rig.probeOwner.mockImplementation(async (record) => { - if (record.sessionId !== 'session-stuck') { - return { outcome: 'pid-absent' } - } - if (!answered) { - answered = true - return { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + it.each([ + ['its stored status, through the settle', true], + ['no stored status, through the restore after the listing', false] + ])( + 'is opened unverified with %s, with no second recovery beside the first, and the rest still settle', + async (_path, rowed) => { + const rig = await newRig() + await crashMidSend(rig, 'session-stuck') + await crashMidSend(rig, 'session-closed', false) + const lease = rig.store.getRecord('session-stuck')!.lease + await rig.crash() + if (!rowed) { + db(rig) + .prepare('DELETE FROM journal_session_state WHERE session_id = ?') + .run('session-stuck') } - hung += 1 - return new Promise(() => {}) - }) - await rig.boot({ startupRecoveryBudgetMs: 50 }) - expect(listedIds(rig)).toEqual(['session-stuck']) - - const done = await Promise.race([ - startup(rig, listedIds(rig)).then(() => 'done'), - new Promise((resolve) => setTimeout(() => resolve('still waiting'), 5_000)) - ]) - - expect(done).toBe('done') - expect(hung).toBe(1) - expect(rig.store.getRecord('session-stuck')!.lease.handoffStage).toBe('recovering') - expect(opened(rig, ['session-closed'])).toEqual(['session-closed']) - expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ - lifecycle: 'idle' - }) - }, 20_000) + // Its provider process was up when Orca died; the check of it answers once, then never again. + await writeOlderBuildLease(rig.root, 'session-stuck', { ...lease }) + let answered = false + let hung = 0 + rig.probeOwner.mockImplementation(async (record) => { + if (record.sessionId !== 'session-stuck') { + return { outcome: 'pid-absent' } + } + if (!answered) { + answered = true + return { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + } + hung += 1 + return new Promise(() => {}) + }) + await rig.boot({ startupRecoveryBudgetMs: 50 }) + expect(listedIds(rig)).toEqual(['session-stuck']) + const warn = vi.spyOn(rig.host.deps.logger, 'warn') + + let background: string[] = [] + const done = await Promise.race([ + runRestTestStartup(rig).then((left) => { + background = left + return 'done' + }), + new Promise((resolve) => setTimeout(() => resolve('still waiting'), 5_000)) + ]) + + expect(done).toBe('done') + expect(background).toEqual(rowed ? [] : ['session-stuck']) + expect(rig.host.hasSession('session-stuck')).toBe(true) + expect(hung).toBe(1) + expect(rig.store.getRecord('session-stuck')!.lease.handoffStage).toBe('recovering') + expect(warn).toHaveBeenCalledWith('a chat recovery outlasted startup; left unverified', { + scope: 'startup-recovery-timeout', + sessionId: 'session-stuck' + }) + expect(opened(rig, ['session-closed'])).toEqual(['session-closed']) + expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ + lifecycle: 'idle' + }) + }, + 20_000 + ) }) describe('a stored status no settle here can clear (R2A-4)', () => { @@ -871,7 +735,7 @@ describe('a stored status no settle here can clear (R2A-4)', () => { }) await rig.boot() - await startup(rig, listedIds(rig)) + await runRestTestStartup(rig) expect(readTestJournalSessionStatus(rig.root, 'session-gone')).toBeNull() expect(opened(rig, ['session-gone'])).toEqual([]) @@ -879,7 +743,7 @@ describe('a stored status no settle here can clear (R2A-4)', () => { // The obligation died: the next boot finds no row to select and opens nothing. await rig.crash() await rig.boot() - await startup(rig, listedIds(rig)) + await runRestTestStartup(rig) expect(readTestJournalSessionStatus(rig.root, 'session-gone')).toBeNull() expect(opened(rig, ['session-gone'])).toEqual([]) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 63580eeeb37c..7f5e9eee441a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -3,18 +3,16 @@ // Leases: one phase checks them, then starts recovering every lease a crash left `recovering` (a // surviving provider process is stopped and its death recorded), so each settle verdict below reads // that evidence; every startup restore answers its lease bookkeeping from that phase. -// Catch-up: a listed chat with history here and no stored status yet (the first launch after the -// upgrade) gets its row from its rows alone, so what follows reads stored status for it too. A listed -// chat whose history is still in a per-chat file is then opened from it, before the listing (see -// `structured-agent-session-startup-per-chat-file-restore`). Seed: every settled chat still listed -// and not open has its status row published from its stored status, so session lists have it at -// paint, without opening the chat. Settle: once every recovery has ended, every chat whose stored -// status shows work a gone process left, listed or not, is opened once, one at a time, and its open -// appends the settlement plan. A listed one goes through the restart restore's own per-chat worker -// and stays open; any other is settled and closed, never indexed or published. Chat commands wait -// for the settle (see `StructuredAgentSessionHostDeps.commandsReady`); listing, paint and status -// reads do not. A listed chat that is corrupt, or whose per-chat file failed to open, is left to the -// background restore after the listing, which is the same worker. +// Seed: every settled chat still listed and not open has its status row published from its stored +// status, so session lists have it at paint, without opening the chat. A listed chat with no current +// row (last written before stored status, by other rules, or by an older build), or whose history is +// still in a per-chat file, is left to the background restore after the listing, which opens it. +// Settle: once every recovery has ended, every chat whose stored status shows work a gone process +// left, listed or not, is opened once, one at a time, and its open appends the settlement plan. A +// listed one goes through the restart restore's own per-chat worker and stays open; any other is +// settled and closed, never indexed or published. Chat commands wait for the settle (see +// `StructuredAgentSessionHostDeps.commandsReady`); listing, paint and status reads do not. A listed +// chat that is corrupt is left to the background restore after the listing, which is the same worker. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionRecord } from '../../../shared/agent-session-record' @@ -36,8 +34,6 @@ import { createStructuredAgentSessionStartupLeasePhase, type StructuredAgentSessionStartupLeasePhase } from './structured-agent-session-startup-lease-phase' -import { restoreListedFromPerChatFiles } from './structured-agent-session-startup-per-chat-file-restore' -import { catchUpMissingStatuses } from './structured-agent-session-startup-status-catch-up' export type StructuredAgentSessionStartupStateDeps = { openDeps: StructuredAgentSessionConversationOpenDeps & { @@ -55,11 +51,10 @@ export type StructuredAgentSessionStartupStateDeps = { * attach or send). */ resolveRecovery: (sessionId: string) => Promise /** The restart restore's per-chat worker (lease bookkeeping, serialize, open, publish), with its - * lease bookkeeping answered by `leases`, at its own concurrency unless given one. */ + * lease bookkeeping answered by `leases`. */ restoreListed: ( records: AgentSessionRecord[], - leases: StructuredAgentSessionStartupLeases, - concurrency?: number + leases: StructuredAgentSessionStartupLeases ) => Promise /** Tests shorten it; production takes the default. */ recoveryBudgetMs?: number @@ -79,11 +74,6 @@ export type StructuredAgentSessionStartupLeases = { export type StructuredAgentSessionStartupState = { /** The startup lease phase's check, which then starts every recovery. Never rejects. */ reconcileRestartLeases: () => Promise - /** Writes the stored status of every listed chat with history here and none yet, before the - * listing answers. Never rejects. */ - catchUpMissingStatuses: (listedIds: readonly string[]) => Promise - /** Opens listed chats still in per-chat files, before the listing answers. Never rejects. */ - restoreListedFromPerChatFiles: (listedIds: readonly string[]) => Promise /** Seeds settled listed chats; answers the listed ids the background restore still opens. */ seedStoredStatuses: (listedIds: readonly string[]) => string[] /** Settles every chat a gone process left with work, once per host. Never rejects. */ @@ -109,14 +99,11 @@ export function createStructuredAgentSessionStartupState( return { reconcileRestartLeases: async () => { await phase.reconciled() - // Started now, awaited by the settle; the listing waits only on its own chats' recoveries. + // Started now, awaited by the settle; the listing never waits on a recovery. if (!deps.openDeps.journalDatabase.readOnly) { void phase.recovered() } }, - catchUpMissingStatuses: (listedIds) => catchUpMissingStatuses(deps, listedIds), - restoreListedFromPerChatFiles: (listedIds) => - restoreListedFromPerChatFiles(deps, listedIds, leases), seedStoredStatuses: (listedIds) => seedStoredStatuses(deps, listedIds), settleOwedSessions: (listedIds) => { settling ??= settleOwedSessions(deps, listedIds, phase, leases) @@ -163,7 +150,7 @@ function seedStoredStatuses( continue } if (!byId.has(sessionId)) { - // Never sent opens nothing; a per-chat file whose open failed is tried again in the background. + // Never sent opens nothing; a chat still in a per-chat file is opened from it in the background. if (hasHistoryOutsideJournalDatabase(database, record)) { background.push(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts deleted file mode 100644 index 0bf80115a704..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.test.ts +++ /dev/null @@ -1,240 +0,0 @@ -// The catch-up's tasks and its quit: a launch where every listed chat has its row does no work at -// all, a launch that derives rows yields only when a task has run long enough, quit stops it with -// nothing more written, for the next launch to redo, and lease recovery starts before it. - -import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import type * as TimersPromises from 'node:timers/promises' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' -import { - closeTestJournalHostDatabases, - createTrackedJournalOpener, - openTestJournalHostDatabase, - readTestJournalSessionStatus -} from '../agent-session-journal/journal-host-database-test-support' -import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' -import { - CORPUS_FENCE, - JOURNAL_SESSION_STATE_CORPUS -} from '../agent-session-journal/journal-session-state-test-corpus' -import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' - -// What the catch-up did, in order: each yield to the event loop, and each fold it finished. -const trace = vi.hoisted(() => { - const log: { events: string[]; afterFold: ((sessionId: string) => void) | null } = { - events: [], - afterFold: null - } - return log -}) - -vi.mock('node:timers/promises', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - setImmediate: (...args: Parameters) => { - trace.events.push('yield') - return actual.setImmediate(...args) - } - } -}) - -vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - foldJournalSessionStatus: async ( - ...args: Parameters - ) => { - const folded = await actual.foldJournalSessionStatus(...args) - trace.events.push(`fold:${args[1]}`) - trace.afterFold?.(args[1]) - return folded - } - } -}) - -import { catchUpMissingStatuses } from './structured-agent-session-startup-status-catch-up' -import { createStructuredAgentSessionStartupState } from './structured-agent-session-startup-state' - -const journals = createTrackedJournalOpener() -let root: string - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'orca-startup-status-catch-up-')) - trace.events = [] - trace.afterFold = null -}) - -afterEach(async () => { - vi.restoreAllMocks() - await journals.closeAll() - closeTestJournalHostDatabases() - await rm(root, { recursive: true, force: true }) -}) - -/** Chats with a settled history in the host's database; `rowless` drops their status rows. */ -async function chats(sessionIds: readonly string[], { rowless }: { rowless: boolean }) { - for (const sessionId of sessionIds) { - const journal = await journals.open({ - identity: { - sessionId, - workspaceId: 'ws-1', - hostId: 'local', - agent: 'codex', - providerHandle: codexProviderHandle(`thread-${sessionId}`) - }, - stateDirectory: root, - mintEpoch: () => `epoch-${sessionId}`, - currentFence: () => CORPUS_FENCE - }) - await JOURNAL_SESSION_STATE_CORPUS.settled(journal) - } - await journals.closeAll() - if (rowless) { - openTestJournalHostDatabase(root).db.exec('DELETE FROM journal_session_state') - } -} - -function deps( - disposed: { value: boolean } = { value: false }, - recovering: { - records: AgentSessionRecord[] - resolveRecovery: (id: string) => Promise - } = { - records: [], - resolveRecovery: async () => true - } -): StructuredAgentSessionStartupStateDeps { - const stand = { - openDeps: { - journalDatabase: openTestJournalHostDatabase(root), - store: { - getRecord: (sessionId: string): AgentSessionRecord => ({ - ...agentSessionRecordFixture(), - sessionId - }), - listRecords: () => recovering.records - }, - logger: { warn: vi.fn(), error: vi.fn() } - }, - canSettle: (candidate: AgentSessionRecord | null): candidate is AgentSessionRecord => - candidate !== null, - seedStatus: vi.fn(), - reconcile: async () => true, - resolveRecovery: recovering.resolveRecovery, - restoreListed: async () => undefined, - serialize: (_sessionId: string, task: () => Promise) => task(), - hasSession: () => false, - isListed: () => true, - isDisposed: () => disposed.value - } - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the catch-up reads only the database, records, logger and the members above. - return stand as unknown as StructuredAgentSessionStartupStateDeps -} - -const yields = () => trace.events.filter((event) => event === 'yield').length -const folds = () => trace.events.filter((event) => event.startsWith('fold:')) - -describe('the startup status catch-up', () => { - it('does nothing on a launch where every listed chat has its row', async () => { - const ids = Array.from({ length: 40 }, (_, index) => `session-${index}`) - await chats(ids, { rowless: false }) - const stand = deps() - const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') - trace.events = [] - - await catchUpMissingStatuses(stand, ids) - - // No fold, no yield, no transaction. - expect(folds()).toEqual([]) - expect(yields()).toBe(0) - expect(transaction).not.toHaveBeenCalled() - }) - - it('yields only once a task has run long enough, not before every chat', async () => { - const ids = Array.from({ length: 30 }, (_, index) => `session-${index}`) - await chats(ids, { rowless: true }) - // Each clock read moves 3 ms: a task reaches the 16 ms budget every few chats. - let clock = 0 - vi.spyOn(performance, 'now').mockImplementation(() => (clock += 3)) - trace.events = [] - - await catchUpMissingStatuses(deps(), ids) - - expect(folds()).toHaveLength(ids.length) - expect(yields()).toBeGreaterThan(0) - expect(yields()).toBeLessThan(ids.length / 2) - expect( - ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId)?.lifecycle === 'idle') - ).toBe(true) - }) - - it.each([ - ['mid-way', 10, 3], - ['during the last fold', 10, 10], - // The 16th fold fills a batch, which is due its write at once. - ['during the fold that fills a batch', 20, 16] - ])( - 'writes nothing after quit %s, and the next launch redoes it', - async (_when, count, quitAtFold) => { - const ids = Array.from({ length: count }, (_, index) => `session-${index}`) - await chats(ids, { rowless: true }) - const disposed = { value: false } - // Every task boundary is due a yield, but the quit lands before the next one. - let clock = 0 - vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) - trace.afterFold = () => { - if (folds().length === quitAtFold) { - disposed.value = true - } - } - const stand = deps(disposed) - const transaction = vi.spyOn(stand.openDeps.journalDatabase, 'transaction') - - await catchUpMissingStatuses(stand, ids) - - expect(folds()).toHaveLength(quitAtFold) - expect(transaction).not.toHaveBeenCalled() - expect(ids.some((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(false) - - trace.afterFold = null - disposed.value = false - await catchUpMissingStatuses(deps(), ids) - expect(ids.every((sessionId) => readTestJournalSessionStatus(root, sessionId))).toBe(true) - } - ) - - it('runs after the lease check started every lease recovery, listed or not', async () => { - const ids = ['session-0', 'session-1'] - await chats(ids, { rowless: true }) - const crashed = ['session-0', 'session-tabless'].map((sessionId) => { - const record = { ...agentSessionRecordFixture(), sessionId } - record.lease = { ...record.lease, sessionId, handoffStage: 'recovering' } - return record - }) - const state = createStructuredAgentSessionStartupState( - deps(undefined, { - records: crashed, - resolveRecovery: async (sessionId) => { - trace.events.push(`recover:${sessionId}`) - return true - } - }) - ) - trace.events = [] - - await state.reconcileRestartLeases() - await state.catchUpMissingStatuses(ids) - - const firstFold = trace.events.findIndex((event) => event.startsWith('fold:')) - expect(firstFold).toBeGreaterThan(-1) - expect(trace.events.slice(0, firstFold)).toEqual( - expect.arrayContaining(['recover:session-0', 'recover:session-tabless']) - ) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts deleted file mode 100644 index ad9264036545..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-status-catch-up.ts +++ /dev/null @@ -1,125 +0,0 @@ -// The first launch after the upgrade to stored status: listed chats with history here and no status -// row get their rows from their rows alone, before the tab listing answers. The seed that follows -// then publishes them, and the settle takes any a gone process left with work, as on every launch. - -import { setImmediate as yieldToEventLoop } from 'node:timers/promises' -import { - foldJournalSessionStatus, - writeJournalSessionStatuses, - type FoldedJournalSessionStatus -} from '../agent-session-journal/journal-session-status-backfill' -import { readJournalSessionStatuses } from '../agent-session-journal/journal-session-state' -import type { StructuredAgentSessionStartupStateDeps } from './structured-agent-session-startup-state' - -// Rows per transaction: one commit per chat rewrites the same table and index pages each time. -const WRITE_BATCH_CHATS = 16 -// Work per main-process task before yielding, so window IPC and the hook server keep answering. -const TASK_BUDGET_MS = 16 - -/** - * Folds and writes the row of every listed chat that has history here and none yet. Never rejects: - * a failure is reported, and the seed, the settle and the listing go on whatever happens here. - */ -export async function catchUpMissingStatuses( - deps: StructuredAgentSessionStartupStateDeps, - listedIds: readonly string[] -): Promise { - const database = deps.openDeps.journalDatabase - if (database.readOnly) { - return - } - try { - const missing = readJournalSessionStatuses(database.db, listedIds).flatMap( - ({ sessionId, status }) => (status ? [] : [sessionId]) - ) - // Only rowless chats are folded; when every listed chat has its row there is no transaction. - if (missing.length > 0) { - await foldAndWrite(deps, missing) - } - } catch (error) { - deps.openDeps.logger.warn('deriving missing chat statuses at startup failed', { - scope: 'startup-status-catch-up', - error - }) - } -} - -/** Quit stops a fold within one part and writes nothing more, for the next launch to redo. */ -async function foldAndWrite( - deps: StructuredAgentSessionStartupStateDeps, - sessionIds: readonly string[] -): Promise { - const quit = new AbortController() - let taskStart = performance.now() - const yieldWhenDue = async (): Promise => { - if (performance.now() - taskStart < TASK_BUDGET_MS) { - return - } - await yieldToEventLoop() - taskStart = performance.now() - if (deps.isDisposed()) { - quit.abort() - } - } - const quitting = () => quit.signal.aborted || deps.isDisposed() - let batch: FoldedJournalSessionStatus[] = [] - const flush = (): void => { - if (!quitting()) { - writeBatch(deps, batch) - } - batch = [] - } - for (const sessionId of sessionIds) { - await yieldWhenDue() - if (quitting()) { - return - } - const record = deps.openDeps.store.getRecord(sessionId) - if (!record || !deps.canSettle(record) || deps.hasSession(sessionId)) { - continue - } - const folded = await foldOne(deps, sessionId, { - yieldTask: yieldWhenDue, - signal: quit.signal - }) - if (folded) { - batch.push(folded) - } - if (batch.length >= WRITE_BATCH_CHATS) { - flush() - } - } - flush() -} - -async function foldOne( - deps: StructuredAgentSessionStartupStateDeps, - sessionId: string, - options: { yieldTask: () => Promise; signal: AbortSignal } -): Promise { - try { - return await foldJournalSessionStatus(deps.openDeps.journalDatabase, sessionId, options) - } catch (error) { - deps.openDeps.logger.warn('deriving a chat status from its rows failed', { - scope: 'startup-status-catch-up', - sessionId, - error - }) - return null - } -} - -/** One transaction; a chat that moved or got a row since its fold is skipped and left to its open. */ -function writeBatch( - deps: StructuredAgentSessionStartupStateDeps, - batch: FoldedJournalSessionStatus[] -): void { - try { - writeJournalSessionStatuses(deps.openDeps.journalDatabase, batch) - } catch (error) { - deps.openDeps.logger.warn('writing chat statuses derived from their rows failed', { - scope: 'startup-status-catch-up', - error - }) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts deleted file mode 100644 index d9c2b72ca36e..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-catch-up.test.ts +++ /dev/null @@ -1,608 +0,0 @@ -// The first launch after the upgrade to stored status: listed chats with history and no status row -// get their rows from their rows alone before the tab listing answers, with no conversation opened. -// The seed then publishes them and the settle takes any a crash left with work, in the same launch. - -import { statSync } from 'node:fs' -import { join } from 'node:path' -import { afterEach, describe, expect, it, vi } from 'vitest' -import type * as JournalSessionStateModule from '../agent-session-journal/journal-session-state' -import { JOURNAL_SESSION_STATUS_RULES } from '../agent-session-journal/journal-session-state' -import type * as StatusBackfillModule from '../agent-session-journal/journal-session-status-backfill' -import type * as PerSessionImportModule from '../agent-session-journal/journal-per-session-import' -import { - closeTestJournalHostDatabases, - insertTestJournalRowJson, - liveTestJournalRows, - loadTestJournal, - openTestJournalHostDatabase, - readTestJournalSessionStatus -} from '../agent-session-journal/journal-host-database-test-support' -import { - createRestTestRig, - restTestChat, - sendRestTestMessage, - type RestTestRig -} from './structured-agent-session-rest-test-rig' -import { StructuredAgentSessionStartupGate } from '../../runtime/structured-agent-session-startup-gate' -import { - latestRestTestStatus, - restTestOpens -} from './structured-agent-session-rest-test-observations' -import { moveRestTestChatToPerChatFile } from './structured-agent-session-rest-test-per-chat-file' - -// The listed chats' stored-status read fails once, when set: the catch-up's own read. -const reads = vi.hoisted(() => { - const control: { failNext: boolean } = { failNext: false } - return control -}) - -vi.mock('../agent-session-journal/journal-session-state', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - readJournalSessionStatuses: (...args: Parameters) => { - if (reads.failNext) { - reads.failNext = false - throw new Error('database is locked') - } - return actual.readJournalSessionStatuses(...args) - } - } -}) - -// The catch-up's work as it happens: fold parts, finished folds and batch commits. -const work = vi.hoisted(() => { - const seen: { - parts: number - folded: string[] - commits: number - /** Runs as each fold ends, before its batch is written. */ - afterFold: ((sessionId: string) => Promise) | null - } = { parts: 0, folded: [], commits: 0, afterFold: null } - return seen -}) - -vi.mock('../agent-session-journal/journal-session-status-backfill', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - foldJournalSessionStatus: async ( - ...[database, sessionId, options]: Parameters - ) => { - const yieldTask = options?.yieldTask - const folded = await actual.foldJournalSessionStatus(database, sessionId, { - ...options, - yieldTask: async () => { - work.parts += 1 - await yieldTask?.() - } - }) - work.parts += 1 - work.folded.push(sessionId) - await work.afterFold?.(sessionId) - return folded - }, - writeJournalSessionStatuses: ( - ...args: Parameters - ) => { - work.commits += 1 - return actual.writeJournalSessionStatuses(...args) - } - } -}) - -// Each preview of a chat still in its per-chat file, and the chats whose file reads as damaged. -const previews = vi.hoisted(() => { - const seen: { sessionIds: string[]; damaged: Set } = { - sessionIds: [], - damaged: new Set() - } - return seen -}) - -vi.mock('../agent-session-journal/journal-per-session-import', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - previewPerSessionJournal: async ( - ...args: Parameters - ) => { - const { sessionId } = args[0].identity - previews.sessionIds.push(sessionId) - if (previews.damaged.has(sessionId)) { - throw new Error('file is not a database') - } - return actual.previewPerSessionJournal(...args) - } - } -}) - -const rigs: RestTestRig[] = [] - -afterEach(async () => { - reads.failNext = false - work.parts = 0 - work.folded.length = 0 - work.commits = 0 - work.afterFold = null - previews.sessionIds.length = 0 - previews.damaged.clear() - for (const rig of rigs.splice(0)) { - await rig.dispose() - } - closeTestJournalHostDatabases() - vi.restoreAllMocks() -}) - -async function newRig(): Promise { - const rig = await createRestTestRig() - rigs.push(rig) - return rig -} - -/** Every chat's row dropped, as the first launch with the status table finds a database. */ -function upgradeToEmptyStatusTable(rig: RestTestRig): void { - openTestJournalHostDatabase(rig.root).db.prepare('DELETE FROM journal_session_state').run() -} - -const listedIds = (rig: RestTestRig) => rig.store.getVisibleSessionTabIndex().sessionIds - -/** Startup as the host's step runs it, through the settle; answers the restore's list. */ -async function startup(rig: RestTestRig): Promise { - const listed = listedIds(rig) - await rig.host.reconcileRestartLeases() - await rig.host.startup.catchUpMissingStatuses(listed) - await rig.host.startup.restoreListedFromPerChatFiles(listed) - const background = rig.host.startup.seedStoredStatuses(listed) - await rig.host.startup.settleOwedSessions(listed) - return background -} - -/** A chat whose turn was still running when Orca died, with or without a tab. */ -async function crashMidTurn(rig: RestTestRig, sessionId: string, listed = true): Promise { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}`, listed }) - const { providerIdentity } = await rig.adapter.dispatch.mock.results.at(-1)!.value - await rig.host - .collaboratorsForTests() - .sessions.get(sessionId)! - .journal.appendItem( - { ...providerIdentity, ordinal: 0 }, - { kind: 'turn', turnId: providerIdentity.turnId, state: 'running', startedAt: 10 }, - { fence: rig.store.getRecord(sessionId)!.lease.runtimeFence, turnScope: { kind: 'thread' } } - ) -} - -const firstStatus = (rig: RestTestRig, sessionId: string) => - rig.statusEvents.findIndex( - (event) => event.type === 'status' && event.session.sessionId === sessionId - ) - -/** The status stream's length when each chat first opened. */ -function recordOpens(rig: RestTestRig): Map { - const openedAt = new Map() - rig.journalOpens.mockImplementation(async (sessionId) => { - if (!openedAt.has(sessionId)) { - openedAt.set(sessionId, rig.statusEvents.length) - } - }) - return openedAt -} - -describe('listed chats with no stored status after the upgrade', () => { - it('get their rows before the seed, from their rows alone, with no chat opened', async () => { - const rig = await newRig() - const ids = Array.from({ length: 12 }, (_, index) => `session-${index}`) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - - const background = await startup(rig) - - expect(background).toEqual([]) - for (const sessionId of ids) { - expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) - expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ status: 'idle' }) - expect(restTestOpens(rig, sessionId)).toBe(0) - } - // Published by the seed, in the listing's order. - const order = ids.map((sessionId) => firstStatus(rig, sessionId)) - expect(order).toEqual(order.toSorted((a, b) => a - b)) - }) - - it('settles an unfinished one in the same launch, before commands go, never showing it from its rows', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-fine', { message: 'done' }) - await crashMidTurn(rig, 'session-running') - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const listed = listedIds(rig) - const openedAt = recordOpens(rig) - await rig.host.reconcileRestartLeases() - - await rig.host.startup.catchUpMissingStatuses(listed) - // Stored, so the settle selects it; not shown: the seed leaves an unsettled row to the settle. - expect(readTestJournalSessionStatus(rig.root, 'session-running')).toMatchObject({ - lifecycle: 'running' - }) - rig.host.startup.seedStoredStatuses(listed) - expect(latestRestTestStatus(rig, 'session-running')).toBeUndefined() - // The settle is what chat commands wait on. - await rig.host.startup.settleOwedSessions(listed) - - expect(restTestOpens(rig, 'session-fine')).toBe(0) - expect(openedAt.has('session-running')).toBe(true) - expect(firstStatus(rig, 'session-running')).toBeGreaterThanOrEqual( - openedAt.get('session-running')! - ) - expect(readTestJournalSessionStatus(rig.root, 'session-running')).toMatchObject({ - lifecycle: 'idle' - }) - expect(latestRestTestStatus(rig, 'session-running')?.status).not.toBe('working') - }) - - it('leaves an unlisted crashed chat with no row showing nothing until its open settles it', async () => { - const rig = await newRig() - await crashMidTurn(rig, 'session-closed', false) - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - expect(listedIds(rig)).not.toContain('session-closed') - - await startup(rig) - - expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toBeNull() - expect(latestRestTestStatus(rig, 'session-closed')).toBeUndefined() - expect(restTestOpens(rig, 'session-closed')).toBe(0) - - // The first read opens it, and the open settles it. - await rig.host.history({ sessionId: 'session-closed', direction: 'tail' }) - expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ - lifecycle: 'idle' - }) - expect(latestRestTestStatus(rig, 'session-closed')?.status).not.toBe('working') - }) - - it('appends few WAL pages for 247 chats: their rows are written 16 to a transaction', async () => { - const rig = await newRig() - const ids = Array.from({ length: 247 }, (_, index) => `session-page-${index}`) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const { db } = openTestJournalHostDatabase(rig.root) - // An empty WAL, so its size after the catch-up is what the catch-up appended. - db.pragma('wal_checkpoint(TRUNCATE)') - const pageSize = Number(db.pragma('page_size', { simple: true })) - - await rig.host.startup.catchUpMissingStatuses(listedIds(rig)) - - const frames = - (statSync(join(rig.root, 'agent-session-journal.db-wal')).size - 32) / (pageSize + 24) - expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) - expect(ids.every((sessionId) => restTestOpens(rig, sessionId) === 0)).toBe(true) - // A commit per chat appends about 2.5 pages each, over 600 for these. - expect(frames).toBeLessThanOrEqual(150) - }, 120_000) - - it('gives a damaged one no row, and its open after the listing fails, logged once, every row kept', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-damaged', { message: 'asked' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - const { db } = openTestJournalHostDatabase(rig.root) - const tip = liveTestJournalRows(db, 'session-damaged').at(-1)! - insertTestJournalRowJson(db, 'session-damaged', tip.seq + 1, '{') - upgradeToEmptyStatusTable(rig) - expect(loadTestJournal(rig.root, 'session-damaged')?.damage).not.toBeNull() - await rig.boot() - const warn = vi.spyOn(rig.host.deps.logger, 'warn') - - const background = await startup(rig) - - expect(readTestJournalSessionStatus(rig.root, 'session-damaged')).toBeNull() - expect(latestRestTestStatus(rig, 'session-damaged')).toBeUndefined() - expect(background).toEqual(['session-damaged']) - await rig.host.restoreReadableSessions(background) - expect(rig.host.hasSession('session-damaged')).toBe(false) - expect( - warn.mock.calls.filter(([, fields]) => fields?.sessionId === 'session-damaged') - ).toHaveLength(1) - expect(readTestJournalSessionStatus(rig.root, 'session-damaged')).toBeNull() - expect(liveTestJournalRows(db, 'session-damaged').length).toBe(tip.seq + 1) - }) - - it('derives a row again on the next launch when its unsynced commit was lost', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-settled', { message: 'asked session-settled' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - await startup(rig) - const first = readTestJournalSessionStatus(rig.root, 'session-settled') - const firstPublished = latestRestTestStatus(rig, 'session-settled') - expect(first).toMatchObject({ lifecycle: 'idle' }) - - // A power cut loses the unsynced commit: the next launch finds no row. - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - const before = rig.statusEvents.length - await startup(rig) - - expect(readTestJournalSessionStatus(rig.root, 'session-settled')).toEqual(first) - expect(latestRestTestStatus(rig, 'session-settled')).toEqual(firstPublished) - expect( - rig.statusEvents - .slice(before) - .every((event) => event.type !== 'status' || event.session.sessionId === 'session-settled') - ).toBe(true) - expect(restTestOpens(rig, 'session-settled')).toBe(0) - }) - - it('lets the seed, the settle and the rest go on when the catch-up fails', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-rowed', { message: 'asked session-rowed' }) - await restTestChat(rig, 'session-rowless', { message: 'asked session-rowless' }) - await crashMidTurn(rig, 'session-crashed') - await rig.host.flushAllStreamedEvents() - await rig.crash() - openTestJournalHostDatabase(rig.root) - .db.prepare('DELETE FROM journal_session_state WHERE session_id = ?') - .run('session-rowless') - await rig.boot() - const warn = vi.spyOn(rig.host.deps.logger, 'warn') - reads.failNext = true - - const background = await startup(rig) - - expect(warn).toHaveBeenCalledWith( - 'deriving missing chat statuses at startup failed', - expect.objectContaining({ scope: 'startup-status-catch-up' }) - ) - // The seed read the rows itself, the rowless chat went to the restore, the settle ran. - expect(latestRestTestStatus(rig, 'session-rowed')).toMatchObject({ status: 'idle' }) - expect(background).toEqual(['session-rowless']) - expect(readTestJournalSessionStatus(rig.root, 'session-crashed')).toMatchObject({ - lifecycle: 'idle' - }) - }) - - it('restores a listed chat still in its per-chat file before the listing answers, read once', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-file', { message: 'asked session-file' }) - await restTestChat(rig, 'session-closed-file', { message: 'asked', listed: false }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - await moveRestTestChatToPerChatFile(rig, 'session-file') - await moveRestTestChatToPerChatFile(rig, 'session-closed-file') - await rig.boot() - - const background = await startup(rig) - - // Restored from its file by the time the listing answers: open, settled and published. - expect(rig.host.hasSession('session-file')).toBe(true) - expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ - latestPrompt: 'asked session-file' - }) - // Not queued for the restore after the listing, which skips it even when given every listed - // chat (its list when startup gave none): one preview, one open. - expect(background).toEqual([]) - await rig.host.restoreReadableSessions(listedIds(rig)) - expect(previews.sessionIds.filter((id) => id === 'session-file')).toEqual(['session-file']) - expect(restTestOpens(rig, 'session-file')).toBe(1) - // One with no tab waits for its first use. - expect(rig.host.hasSession('session-closed-file')).toBe(false) - expect(previews.sessionIds).not.toContain('session-closed-file') - expect(latestRestTestStatus(rig, 'session-closed-file')).toBeUndefined() - }) - - it('isolates a damaged per-chat file and a missing one: the others restore and startup goes on', async () => { - const rig = await newRig() - for (const sessionId of ['session-damaged', 'session-missing', 'session-file']) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.host.flushAllStreamedEvents() - await rig.crash() - await moveRestTestChatToPerChatFile(rig, 'session-damaged') - await moveRestTestChatToPerChatFile(rig, 'session-file') - // Its file is gone too: nothing of it anywhere but its record and tab. - for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { - openTestJournalHostDatabase(rig.root) - .db.prepare(`DELETE FROM ${table} WHERE session_id = ?`) - .run('session-missing') - } - previews.damaged.add('session-damaged') - await rig.boot() - const warn = vi.spyOn(rig.host.deps.logger, 'warn') - - const background = await startup(rig) - - expect(listedIds(rig)).toEqual(['session-damaged', 'session-missing', 'session-file']) - expect(rig.host.hasSession('session-file')).toBe(true) - expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ status: 'idle' }) - expect(warn).toHaveBeenCalledWith( - 'restoring a chat for reading failed', - expect.objectContaining({ sessionId: 'session-damaged' }) - ) - expect(rig.host.hasSession('session-damaged')).toBe(false) - // The damaged file is tried again after the listing; the chat with no file anywhere has nothing - // to read, and is listed all the same. - expect(background).toEqual(['session-damaged']) - }) - it('gives a chat whose tab closes during the catch-up its row, but no status row', async () => { - const rig = await newRig() - for (const sessionId of ['session-kept', 'session-closing']) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.host.flushAllStreamedEvents() - await rig.crash() - upgradeToEmptyStatusTable(rig) - await rig.boot() - let closedAt = -1 - // A close is not held for startup: it lands between the catch-up's folds. - work.afterFold = async (sessionId) => { - if (sessionId === 'session-kept') { - await rig.host.setSessionTabVisibility('session-closing', false) - await rig.host.close('session-closing', 'user-close') - closedAt = rig.statusEvents.length - } - } - - await startup(rig) - - expect(closedAt).toBeGreaterThan(-1) - expect(readTestJournalSessionStatus(rig.root, 'session-closing')).toMatchObject({ - lifecycle: 'idle' - }) - expect( - rig.statusEvents - .slice(closedAt) - .filter((event) => event.type === 'status' && event.session.sessionId === 'session-closing') - ).toEqual([]) - expect(restTestOpens(rig, 'session-closing')).toBe(0) - expect(latestRestTestStatus(rig, 'session-kept')).toMatchObject({ status: 'idle' }) - }) - - it('lets a send through mid-catch-up once the gate ceiling passes, behind at most one fold part and one batch commit', async () => { - const rig = await newRig() - const ids = Array.from({ length: 6 }, (_, index) => `session-long-${index}`) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - const journal = rig.host.collaboratorsForTests().sessions.get(sessionId)!.journal - const fence = rig.store.getRecord(sessionId)!.lease.runtimeFence - // Long enough that its fold takes several parts, as a long chat's does. - for (let index = 0; index < 1_200; index += 1) { - await journal.appendItem( - { provider: 'orca', clientMessageId: `${sessionId}-note-${index}` }, - { kind: 'status', text: `${index} ${'x'.repeat(400)}` }, - { fence, turnScope: { kind: 'thread' } } - ) - } - } - await restTestChat(rig, 'session-send', { message: 'first' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - // The chat the send goes to keeps its row; the long ones are the catch-up's. - openTestJournalHostDatabase(rig.root) - .db.prepare('DELETE FROM journal_session_state WHERE session_id != ?') - .run('session-send') - const gate = new StructuredAgentSessionStartupGate(30) - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - gate.hold() - await rig.boot({ commandsReady: gate.ready }) - // Every task's budget is spent at its first check, so each fold part is a task of its own. - let clock = 0 - vi.spyOn(performance, 'now').mockImplementation(() => (clock += 100)) - let sent: Promise | null = null - let ahead = { parts: -1, commits: -1 } - // The catch-up's progress when the send was answered. - let answeredAt = { folded: -1, commits: -1 } - work.afterFold = async (sessionId) => { - if (sessionId !== ids[0]) { - return - } - // The catch-up outlasts the gate's ceiling, which lets commands go. - await new Promise((resolve) => setTimeout(resolve, 60)) - expect(gate.ready()).toBeNull() - const before = { parts: work.parts, commits: work.commits } - // A send arrives as a message: its own task, which runs once the catch-up yields. - sent = new Promise((resolve) => { - setImmediate(() => { - ahead = { parts: work.parts - before.parts, commits: work.commits - before.commits } - resolve( - sendRestTestMessage(rig, 'session-send', 'during the catch-up').then((answer) => { - answeredAt = { folded: work.folded.length, commits: work.commits } - return answer - }) - ) - }) - }) - } - - await rig.host.startup.catchUpMissingStatuses(listedIds(rig)) - - expect(await sent).toMatchObject({ ok: true }) - expect(ahead.parts).toBeGreaterThanOrEqual(0) - expect(ahead.parts).toBeLessThanOrEqual(1) - expect(ahead.commits).toBeLessThanOrEqual(1) - // Answered while the catch-up still ran: before its last fold and its last commit. - expect(answeredAt.folded).toBeGreaterThan(0) - expect(answeredAt.folded).toBeLessThan(ids.length) - expect(answeredAt.commits).toBeLessThan(work.commits) - expect(ids.every((sessionId) => readTestJournalSessionStatus(rig.root, sessionId))).toBe(true) - }, 120_000) -}) - -describe('a stored status derived by other rules', () => { - /** The chat's row as a build with other derivation rules left it. */ - function deriveByOtherRules(rig: RestTestRig, sessionId: string, lifecycle = 'idle'): void { - openTestJournalHostDatabase(rig.root) - .db.prepare( - `UPDATE journal_session_state SET rules_version = ?, lifecycle = ?, - summary_json = '{"status":"idle","latestPrompt":"by other rules"}' WHERE session_id = ?` - ) - .run(JOURNAL_SESSION_STATUS_RULES - 1, lifecycle, sessionId) - } - - const rulesOf = (rig: RestTestRig, sessionId: string) => - openTestJournalHostDatabase(rig.root) - .db.prepare('SELECT rules_version FROM journal_session_state WHERE session_id = ?') - .get(sessionId)?.rules_version - - it('is derived again for a listed chat exactly as a missing row is, and its old row is never shown', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-kept', { message: 'asked session-kept' }) - await restTestChat(rig, 'session-other-rules', { message: 'asked session-other-rules' }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - deriveByOtherRules(rig, 'session-other-rules') - await rig.boot() - - const background = await startup(rig) - - // The rowless path: one fold, one batched write, no open. - expect(work.folded).toEqual(['session-other-rules']) - expect(work.commits).toBe(1) - expect(restTestOpens(rig, 'session-other-rules')).toBe(0) - expect(background).toEqual([]) - expect(rulesOf(rig, 'session-other-rules')).toBe(JOURNAL_SESSION_STATUS_RULES) - expect(latestRestTestStatus(rig, 'session-other-rules')).toMatchObject({ - status: 'idle', - latestPrompt: 'asked session-other-rules' - }) - }) - - it('is left alone for a chat with no tab until it opens, and never selects it to settle', async () => { - const rig = await newRig() - await restTestChat(rig, 'session-closed', { message: 'asked session-closed', listed: false }) - await rig.host.flushAllStreamedEvents() - await rig.crash() - // By other rules it would be owed: this build does not trust that. - deriveByOtherRules(rig, 'session-closed', 'running') - await rig.boot() - - await startup(rig) - - expect(work.folded).toEqual([]) - expect(restTestOpens(rig, 'session-closed')).toBe(0) - expect(rulesOf(rig, 'session-closed')).toBe(JOURNAL_SESSION_STATUS_RULES - 1) - expect(latestRestTestStatus(rig, 'session-closed')).toBeUndefined() - - // Its open writes the row by these rules. - await rig.host.history({ sessionId: 'session-closed', direction: 'tail' }) - expect(rulesOf(rig, 'session-closed')).toBe(JOURNAL_SESSION_STATUS_RULES) - expect(readTestJournalSessionStatus(rig.root, 'session-closed')).toMatchObject({ - lifecycle: 'idle', - summary: { latestPrompt: 'asked session-closed' } - }) - }) -}) diff --git a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts index 467409f50be9..0ac2fcb55649 100644 --- a/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts +++ b/src/main/runtime/orca-runtime-restore-structured-agent-session-tabs-once.ts @@ -88,10 +88,9 @@ export class OrcaRuntimeWithRestoreStructuredAgentSessionTabsOnce extends OrcaRu for (const session of restored) { this.projectStructuredAgentSessionTab({ ...session, ...quiet }) } - // Startup already seeded each settled listed chat not yet open, opened listed chats still in - // per-chat files, and is settling the ones a gone process left with work; this opens only what - // stored status could not answer (a row this build cannot read, a corrupt history, a per-chat - // file whose open failed). + // Startup already seeded each settled listed chat not yet open, and is settling the ones a gone + // process left with work; this opens only what stored status could not answer (no current row, + // a corrupt history, a chat still in a per-chat file). // Whoever answers with this list starts it, once that answer is out. const background = this.structuredAgentSessionBackgroundRestoreIds ?? listedIds this.owedStructuredAgentSessionHistoryRestore = host diff --git a/src/main/runtime/orca-runtime-structured-session-restore.test.ts b/src/main/runtime/orca-runtime-structured-session-restore.test.ts index f98efe27789d..c0f5dc1fb816 100644 --- a/src/main/runtime/orca-runtime-structured-session-restore.test.ts +++ b/src/main/runtime/orca-runtime-structured-session-restore.test.ts @@ -22,8 +22,6 @@ type StartupHostMembers = Partial< function installStartupHost(members: StartupHostMembers): void { const host = { startup: { - catchUpMissingStatuses: async () => undefined, - restoreListedFromPerChatFiles: async () => undefined, seedStoredStatuses: (ids: readonly string[]) => [...ids], settleOwedSessions: async () => undefined }, diff --git a/src/main/runtime/structured-agent-session-startup-step.ts b/src/main/runtime/structured-agent-session-startup-step.ts index 2d64a22deea5..3a778e64df11 100644 --- a/src/main/runtime/structured-agent-session-startup-step.ts +++ b/src/main/runtime/structured-agent-session-startup-step.ts @@ -1,5 +1,4 @@ // The structured-chat step of host startup, before any client lists a tab: the restart lease check, -// the rows listed chats lack derived from their rows, listed chats still in per-chat files opened, // each listed chat's status seeded from its stored status, and every chat a gone process left with // work settled. The tab list waits for all but the settle; chat commands wait for the settle too. @@ -38,11 +37,6 @@ export async function runStructuredAgentSessionStartupStep( ): Promise { await host.reconcileRestartLeases() const listedIds = listedStructuredAgentSessionIds(host, savedSession) - // Before the listing answers: the first launch after the upgrade derives the rows listed chats - // lack, so the seed and the settle below read stored status for every one of them. - await host.startup.catchUpMissingStatuses(listedIds) - // Listed chats an older build left in per-chat files have no stored status to list them from. - await host.startup.restoreListedFromPerChatFiles(listedIds) const background = host.startup.seedStoredStatuses(listedIds) // Not awaited here: the tab list and paint never wait on it; chat commands do. onSettling(host.startup.settleOwedSessions(listedIds)) diff --git a/src/main/runtime/structured-agent-session-support-probe.test.ts b/src/main/runtime/structured-agent-session-support-probe.test.ts index a116ba5d8ad0..14ef19df1c32 100644 --- a/src/main/runtime/structured-agent-session-support-probe.test.ts +++ b/src/main/runtime/structured-agent-session-support-probe.test.ts @@ -36,8 +36,6 @@ function stubStructuredHostInstall(runtime: OrcaRuntimeService): { const host = { reconcileRestartLeases: vi.fn(async () => {}), startup: { - catchUpMissingStatuses: vi.fn(async () => undefined), - restoreListedFromPerChatFiles: vi.fn(async () => undefined), seedStoredStatuses: vi.fn((ids: readonly string[]) => [...ids]), settleOwedSessions: vi.fn(async () => undefined) }, diff --git a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts index 997a54935000..4a01c9c156d1 100644 --- a/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts +++ b/src/main/runtime/structured-agent-session-tab-listing-startup.test.ts @@ -1,54 +1,24 @@ // The chat tab list a client asks for at startup answers from records and the tab table. It waits // for no chat's history; each chat's history is read after the answer. -import { rm, writeFile } from 'node:fs/promises' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../shared/protocol-version' import type { RuntimeMobileSessionTabsResult } from '../../shared/runtime-types' import { closeTestJournalHostDatabases, - openTestJournalHostDatabase, - readTestJournalSessionStatus + openTestJournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database-test-support' -import type * as StatusBackfillModule from '../native-chat/agent-session-journal/journal-session-status-backfill' import { setStructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-registry' import { createRestTestRig, restTestChat, type RestTestRig } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-rig' -import { - latestRestTestStatus, - restTestOpens -} from '../native-chat/agent-session-wire/structured-agent-session-rest-test-observations' -import { moveRestTestChatToPerChatFile } from '../native-chat/agent-session-wire/structured-agent-session-rest-test-per-chat-file' -import { legacyJournalDatabaseFile } from '../native-chat/agent-session-journal/journal-paths' import { AgentSessionStoreTransactions } from './agent-session-store-transactions' import { OrcaRuntimeService } from './orca-runtime' import { RpcDispatcher } from './rpc/dispatcher' import { SESSION_TAB_METHODS } from './rpc/methods/session-tabs' -// Chats whose status cannot be computed from their rows, as an unreadable database page fails it. -const failingDerives = vi.hoisted(() => new Set()) - -vi.mock( - '../native-chat/agent-session-journal/journal-session-status-backfill', - async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - foldJournalSessionStatus: async ( - ...args: Parameters - ) => { - if (failingDerives.has(args[1])) { - throw new Error('EACCES: permission denied') - } - return actual.foldJournalSessionStatus(...args) - } - } - } -) - type ListingInternals = { store: { getWorkspaceSession: () => unknown } getClientSettings(): { experimentalStructuredNativeChat: boolean } @@ -76,7 +46,6 @@ beforeEach(async () => { afterEach(async () => { releaseHeld() - failingDerives.clear() // The listing starts the history restore after its answer, without awaiting it; it finishes // before the rig removes its files. await new Promise((resolve) => setImmediate(resolve)) @@ -140,11 +109,6 @@ function restartedRuntime(workspaceSession: unknown = null) { return { runtime, listAll, subscribeAll } } -/** The files an older build left: no chat has stored state, so each gets it after the answer. */ -function forgetStoredChatState(): void { - openTestJournalHostDatabase(rig.root).db.exec('DELETE FROM journal_session_state') -} - function chatIds(snapshots: RuntimeMobileSessionTabsResult[]): string[] { return snapshots.flatMap((snapshot) => snapshot.tabs.flatMap((tab) => (tab.type === 'agent-session' ? [tab.sessionId] : [])) @@ -160,61 +124,6 @@ function within(promise: Promise, ms = 2_000): Promise { - it('answers once each listed chat with no stored status has one, published, opening none (T1)', async () => { - const ids = ['session-1', 'session-2', 'session-3'] - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: sessionId }) - } - await rig.crash() - await rig.boot() - forgetStoredChatState() - const { listAll } = restartedRuntime() - - expect(await within(listAll())).toEqual(ids) - - // Derived from their rows before the answer, and published by the seed that follows. - for (const sessionId of ids) { - expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) - expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ latestPrompt: sessionId }) - expect(restTestOpens(rig, sessionId)).toBe(0) - } - }) - - it('lists chats still in per-chat files, restored before the answer, a damaged or missing file included', async () => { - const ids = ['session-file', 'session-damaged', 'session-missing'] - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: sessionId }) - } - await rig.crash() - await moveRestTestChatToPerChatFile(rig, 'session-file') - await moveRestTestChatToPerChatFile(rig, 'session-damaged') - const damaged = legacyJournalDatabaseFile( - openTestJournalHostDatabase(rig.root).legacyDirectoryFor({ - workspaceId: rig.store.getRecord('session-damaged')!.location.workspaceId, - sessionId: 'session-damaged' - }) - ) - await writeFile(damaged, 'not a database') - await rm(`${damaged}-wal`, { force: true }) - for (const table of ['journal_rows', 'journal_sessions', 'journal_session_state']) { - openTestJournalHostDatabase(rig.root) - .db.prepare(`DELETE FROM ${table} WHERE session_id = ?`) - .run('session-missing') - } - await rig.boot() - vi.spyOn(rig.host.deps.logger, 'warn').mockImplementation(() => undefined) - const { listAll } = restartedRuntime() - - expect(await within(listAll())).toEqual(ids) - - // Restored from its file before the answer; the others are listed and block nothing. - expect(rig.host.hasSession('session-file')).toBe(true) - expect(latestRestTestStatus(rig, 'session-file')).toMatchObject({ - latestPrompt: 'session-file' - }) - expect(rig.host.hasSession('session-damaged')).toBe(false) - }) - it.each(['listAll', 'subscribeAll'] as const)( 'starts the history restore only after %s has answered, census and all', async (method) => { @@ -224,23 +133,20 @@ describe('listing chat tabs at startup', () => { } await rig.crash() await rig.boot() - forgetStoredChatState() + // No stored status: every chat is left to the restore, which opens it. + openTestJournalHostDatabase(rig.root).db.exec('DELETE FROM journal_session_state') const restore = vi.spyOn(rig.host, 'restoreReadableSessions') const { runtime, listAll, subscribeAll } = restartedRuntime() - const rows = () => - ids.filter((id) => readTestJournalSessionStatus(rig.root, id) !== null).length const seen = () => ({ started: restore.mock.calls.length, - opens: rig.journalOpens.mock.calls.length, - rows: rows() + opens: rig.journalOpens.mock.calls.length }) const answered = method === 'listAll' ? { ids: await listAll(), atAnswer: seen() } : await subscribeAll(seen) - // Every row is written before the answer; nothing is opened before or after it. - expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0, rows: ids.length } }) - await vi.waitFor(() => expect(restore).toHaveBeenCalledOnce()) + expect(answered).toEqual({ ids, atAnswer: { started: 0, opens: 0 } }) + await vi.waitFor(() => expect(restore).toHaveBeenCalledWith(ids)) runtime.cleanupSubscriptionsForConnection('connection-1') } ) @@ -270,27 +176,6 @@ describe('listing chat tabs at startup', () => { } ) - it('answers with a /clear committed after an earlier listing', async () => { - await restTestChat(rig, 'session-cleared') - await restTestChat(rig, 'session-fresh', { listed: false }) - await rig.crash() - await rig.boot() - const { listAll } = restartedRuntime() - expect(await listAll()).toEqual(['session-cleared']) - - const record = rig.store.getRecord('session-cleared')! - await rig.store.setConversationCommand('session-cleared', record.lease.runtimeFence, { - command: 'clear', - state: 'completed', - phase: 'committed', - operationId: 'clear-1', - callerKey: 'client-1', - replacementSessionId: 'session-fresh' - }) - - expect(await listAll()).toEqual(['session-fresh']) - }) - it('answers with a cleared chat revealed while the listing waits on its census', async () => { await restTestChat(rig, 'session-cleared') await restTestChat(rig, 'session-fresh', { listed: false }) @@ -358,53 +243,6 @@ describe('listing chat tabs at startup', () => { expect(writes).not.toHaveBeenCalled() }) - it('lists every chat when four fail to read, and the others still get status rows (T5)', async () => { - // The first four failing: each failure must cost only its own chat, never the chats after it. - // Their rows cannot be read for a status, and their open, which they are left to, fails too. - const ids = ['session-1', 'session-2', 'session-3', 'session-4', 'session-5', 'session-6'] - const failing = ids.slice(0, 4) - for (const sessionId of ids) { - await restTestChat(rig, sessionId, { message: `asked ${sessionId}` }) - } - await rig.crash() - await rig.boot() - forgetStoredChatState() - for (const sessionId of failing) { - failingDerives.add(sessionId) - } - rig.journalOpens.mockImplementation(async (sessionId) => { - if (failing.includes(sessionId)) { - throw new Error('EACCES: permission denied') - } - }) - const warn = vi.spyOn(rig.host.deps.logger, 'warn').mockImplementation(() => undefined) - const restore = vi.spyOn(rig.host, 'restoreReadableSessions') - const { listAll } = restartedRuntime() - - expect(await within(listAll())).toEqual(ids) - await vi.waitFor(() => expect(restore).toHaveBeenCalledOnce()) - await restore.mock.results[0]?.value - for (const sessionId of ['session-5', 'session-6']) { - expect(readTestJournalSessionStatus(rig.root, sessionId)).toMatchObject({ lifecycle: 'idle' }) - expect(latestRestTestStatus(rig, sessionId)).toMatchObject({ - latestPrompt: `asked ${sessionId}` - }) - } - const failures = warn.mock.calls.filter( - ([message]) => message === 'restoring a chat for reading failed' - ) - expect(failures.map(([, detail]) => detail)).toEqual( - failing.map((sessionId) => - expect.objectContaining({ - sessionId, - error: expect.objectContaining({ message: 'EACCES: permission denied' }) - }) - ) - ) - expect(await listAll()).toEqual(ids) - expect(restore).toHaveBeenCalledOnce() - }) - it('lists in tab-table order, not the order the chats were created (T7)', async () => { for (const sessionId of ['session-c', 'session-a', 'session-b']) { await restTestChat(rig, sessionId, { listed: false }) diff --git a/src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx b/src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx deleted file mode 100644 index fa1e6e9f6482..000000000000 --- a/src/renderer/src/runtime/web-session-tabs-sync-slow-initial-listing.test.tsx +++ /dev/null @@ -1,200 +0,0 @@ -// @vitest-environment happy-dom - -// A paired host's first tab listing can take longer than the client waits for it: on the first -// boot after an upgrade the host derives its chats' statuses before it answers. The listing's -// timeout must not leave the client without the host's tabs. The all-worktrees stream, open since -// the same moment, delivers the host's full inventory as its first frame once the host is ready. - -import { act, cleanup, renderHook } from '@testing-library/react' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { RuntimeMobileSessionTabsResult } from '../../../shared/runtime-types' -import type { RuntimeRpcResponse } from '../../../shared/runtime-rpc-envelope' -import type * as WorktreeRuntimeOwnerModule from '@/lib/worktree-runtime-owner' - -const mocks = vi.hoisted(() => ({ - getExplicitRuntimeEnvironmentIdForWorktree: vi.fn(), - recoverSnapshot: vi.fn(), - runtimeSessionMirrorEnvironmentKey: vi.fn() -})) - -vi.mock('./use-runtime-session-mirror-environment-key', () => ({ - useRuntimeSessionMirrorEnvironmentKeys: () => ({ - environmentKey: mocks.runtimeSessionMirrorEnvironmentKey(), - resubscribeSignal: '' - }) -})) - -vi.mock('@/lib/worktree-runtime-owner', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - getExplicitRuntimeEnvironmentIdForWorktree: mocks.getExplicitRuntimeEnvironmentIdForWorktree - } -}) - -vi.mock('./web-session-terminal-orphan-recovery', () => ({ - recoverWebSessionTerminalOrphansBeforeApply: mocks.recoverSnapshot -})) - -import { useAppStore } from '@/store' -import type { PublicKnownRuntimeEnvironment } from '../../../shared/runtime-environments' -import type { AppState } from '@/store/types' -import { replaceRuntimeEnvironmentRevisions } from './runtime-environment-revision' -import { - resetWebSessionTabsSnapshotFreshnessForTests, - useWebSessionTabsSync -} from './web-session-tabs-sync' - -const ENV_A = 'env-a' -const WORKTREE = 'repo-a::worktree-a' -const REVISION_A = 101 -const MIRROR_KEY = `${ENV_A}\u0001runtime-a\u00011\u0001${REVISION_A}` -const initialState = useAppStore.getInitialState() - -type RuntimeCall = typeof window.api.runtimeEnvironments.call -type RuntimeSubscribe = typeof window.api.runtimeEnvironments.subscribe -type RuntimeSubscription = { - request: Parameters[0] - callbacks: Parameters[1] -} - -const subscriptions: RuntimeSubscription[] = [] -// The listing in flight, answered by the test: here, only ever by the client's own timeout. -let answerListing: (response: RuntimeRpcResponse) => void = () => {} -const runtimeCall = vi.fn( - (request) => - new Promise((resolve) => { - if (request.method === 'session.tabs.listAll') { - answerListing = resolve - } - }) -) -const runtimeSubscribe = vi.fn(async (request, callbacks) => { - subscriptions.push({ request, callbacks }) - return { unsubscribe: vi.fn(), sendBinary: vi.fn() } -}) - -function chatSnapshot(): RuntimeMobileSessionTabsResult { - return { - worktree: WORKTREE, - publicationEpoch: 'structured:epoch-1', - snapshotVersion: 1, - activeGroupId: 'host-group-1', - activeTabId: 'agent-session:codex-1', - activeTabType: 'agent-session', - tabs: [ - { - type: 'agent-session', - id: 'agent-session:codex-1', - title: 'Codex Chat', - sessionId: 'codex-1', - agent: 'codex', - isActive: true - } - ] - } -} - -async function settle(): Promise { - for (let index = 0; index < 6; index += 1) { - await Promise.resolve() - } -} - -function inventoryStream(): RuntimeSubscription { - const stream = subscriptions.find( - ({ request }) => request.method === 'session.tabs.subscribeAll' && request.selector === ENV_A - ) - if (!stream) { - throw new Error('the all-worktrees stream was never opened') - } - return stream -} - -const mirroredChats = () => - (useAppStore.getState().unifiedTabsByWorktree[WORKTREE] ?? []).filter( - (tab) => tab.contentType === 'agent-session' - ) - -describe('a paired host whose first tab listing outlasts the client', () => { - beforeEach(() => { - subscriptions.length = 0 - runtimeCall.mockClear() - runtimeSubscribe.mockClear() - mocks.recoverSnapshot.mockReset().mockImplementation(async (_state, snapshot) => snapshot) - mocks.getExplicitRuntimeEnvironmentIdForWorktree.mockReset().mockReturnValue(ENV_A) - mocks.runtimeSessionMirrorEnvironmentKey.mockReset().mockReturnValue(MIRROR_KEY) - Object.defineProperty(window, 'api', { - configurable: true, - value: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } } - }) - resetWebSessionTabsSnapshotFreshnessForTests() - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mirror scan and revision ledger read only id, createdAt and pairingRevision. - const runtimeEnvironments = [ - { id: ENV_A, createdAt: 100, pairingRevision: REVISION_A } - ] as PublicKnownRuntimeEnvironment[] - replaceRuntimeEnvironmentRevisions(runtimeEnvironments) - useAppStore.setState( - { - ...initialState, - activeWorktreeId: WORKTREE, - workspaceSessionReady: true, - runtimeEnvironments, - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mirror reads only runtimeId and connectionGeneration from each status. - runtimeStatusByEnvironmentId: new Map([ - [ENV_A, { status: { runtimeId: 'runtime-a' }, connectionGeneration: 1 }] - ]) as AppState['runtimeStatusByEnvironmentId'] - }, - true - ) - }) - - afterEach(() => { - cleanup() - useAppStore.setState(initialState, true) - replaceRuntimeEnvironmentRevisions([]) - resetWebSessionTabsSnapshotFreshnessForTests() - }) - - it("still mirrors the host's chats from the stream after the listing times out", async () => { - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - renderHook(() => useWebSessionTabsSync()) - await act(settle) - expect(runtimeCall).toHaveBeenCalledWith( - expect.objectContaining({ method: 'session.tabs.listAll', timeoutMs: 15_000 }) - ) - const stream = inventoryStream() - - // 15 s pass with the host still deriving: the client gives up on its listing. - await act(async () => { - answerListing({ - id: 'list-all', - ok: false, - error: { - code: 'runtime_timeout', - message: 'Timed out waiting for the remote Orca runtime response.' - }, - _meta: { runtimeId: 'runtime-a' } - }) - await settle() - }) - expect(mirroredChats()).toEqual([]) - - // The host finishes: the stream's first frame is its whole inventory. - await act(async () => { - stream.callbacks.onResponse({ - id: 'subscription-event', - ok: true, - result: { type: 'snapshots', snapshots: [chatSnapshot()], authoritative: true }, - _meta: { runtimeId: 'runtime-a' } - }) - await settle() - }) - - expect(mirroredChats().map((tab) => tab.entityId)).toEqual(['codex-1']) - // Nothing retried the listing: the stream alone carried the tabs. - expect( - runtimeCall.mock.calls.filter(([request]) => request.method === 'session.tabs.listAll') - ).toHaveLength(1) - }) -}) diff --git a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts index 06a08535ff68..870e48454f7d 100644 --- a/tests/e2e/structured-session-reopened-tab-close.unit.test.ts +++ b/tests/e2e/structured-session-reopened-tab-close.unit.test.ts @@ -129,8 +129,6 @@ async function setup() { { reconcileRestartLeases: async () => undefined, startup: { - catchUpMissingStatuses: async () => undefined, - restoreListedFromPerChatFiles: async () => undefined, seedStoredStatuses: (ids: readonly string[]) => [...ids], settleOwedSessions: async () => undefined }, From 20769bf3d8d47f6630dccd98260942bd9a88c9d9 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:42:27 -0700 Subject: [PATCH 103/107] test(native-chat): one corpus walk checks the write path and startup's reading of each row; the failing-savepoint case tests its rollback The stored-status suite walked the corpus several times for one property. The fold-undo walk already checks, after every real commit, that the stored row equals a fresh derivation; it now also checks at the end of each chat what startup reads: selection against the pinned answer, the partial-index read, the active turn, and a settled summary no fence changes. The separate T3 and T15a walks and the schema-version test (the records-migration test pins version 4) are gone. A new corpus case, a failed start that rejects queued sends, carries the append-status-commit test's failed-COMMIT case into the same walk; the rules digest is re-recorded for it with no rules bump, since no derivation changed. The bookkeeping-savepoint case inserted a status row missing the columns that became NOT NULL, so the INSERT threw before writing and "the savepoint rolled back its write" passed without being tested. It now inserts a complete row, checks the write happened, and runs once on the chat that holds every case. --- .../journal-append-status-commit.test.ts | 22 +---- .../journal-fold-undo.test.ts | 99 +++++++++++++------ .../journal-session-state-test-corpus.ts | 9 ++ .../journal-session-state-test-writes.ts | 22 +++++ .../journal-session-state.test.ts | 57 ----------- .../journal-session-status-rules.test.ts | 5 +- 6 files changed, 106 insertions(+), 108 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts b/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts index c505e0974fc1..38c857f42fbe 100644 --- a/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-append-status-commit.test.ts @@ -189,6 +189,7 @@ describe("a send's ledger answer", () => { }) }) +// A failed COMMIT of this batch is in the fold-undo property, through the corpus's failed start. describe("a failed start's rows", () => { it('commit with one status write, from the fold that holds every row', async () => { const journal = await openChat() @@ -202,25 +203,4 @@ describe("a failed start's rows", () => { expect(stored()).toEqual(freshDerivation()) expect(stored()).toMatchObject({ queuedSends: 0 }) }) - - it('leave no status write and no fold change when the COMMIT fails', async () => { - const journal = await openChat() - await send(journal, 'first') - await send(journal, 'second') - const tip = journal.cursor() - const before = stored() - const writesBefore = statusWrites() - failNextCommit() - - await expect(journal.appendLifecycleBatch(startFailureBatch())).rejects.toThrow('COMMIT failed') - - expect(statusWrites()).toBe(writesBefore) - expect(stored()).toEqual(before) - expect(journal.cursor()).toEqual(tip) - expect(journal.submissions().map((entry) => entry.dispatchState)).toEqual([ - 'pending', - 'pending' - ]) - expect(journal.snapshot()).toEqual(renderJournalState(loadTestJournal(root, SESSION)!.state)) - }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts index 9f59d2fa2716..ff1961d7b6ca 100644 --- a/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-fold-undo.test.ts @@ -1,8 +1,13 @@ -// A failed append's undo, as a property: every append first runs with a failing COMMIT, then for -// real, and after each the in-memory fold equals a fresh replay of the disk, container by container -// with key order, scalars and turn scope, and the stored status equals a fresh derivation. +// The write path, as a property: every append first runs with a failing COMMIT, then for real, and +// after each the in-memory fold equals a fresh replay of the disk, container by container with key +// order, scalars and turn scope, and the stored status equals a fresh derivation. At the end of each +// chat, startup's selection reads the pinned answer. +// +// The harness wraps the store's private row writer: a writer refactor breaks it loudly. import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { activeStructuredAgentSessionTurnIdBySequence } from '../../../shared/structured-agent-session-live-turn' +import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -19,12 +24,22 @@ import { } from './journal-host-database-test-support' import type { JournalFoldHolder } from './journal-fold-holder' import { JournalQueuedMessages } from './journal-queued-messages' -import { MAX_JOURNAL_APPLIED_SETTLEMENT_IDS, type JournalReducerState } from './journal-reducer' -import { deriveJournalSessionStatus } from './journal-session-state' +import { + MAX_JOURNAL_APPLIED_SETTLEMENT_IDS, + renderJournalState, + type JournalReducerState +} from './journal-reducer' +import { + deriveJournalSessionStatus, + isUnsettledJournalSessionStatus, + readUnsettledJournalSessionIds +} from './journal-session-state' import { CORPUS_FENCE, + CORPUS_UNSETTLED, JOURNAL_SESSION_STATE_CASES, - JOURNAL_SESSION_STATE_CORPUS + JOURNAL_SESSION_STATE_CORPUS, + type JournalSessionStateCase } from './journal-session-state-test-corpus' import type { AgentSessionJournal } from './journal-store' @@ -172,12 +187,37 @@ function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { const THREAD = { fence: CORPUS_FENCE, turnScope: { kind: 'thread' as const } } +/** What startup reads of the chat's final row: selection, the partial index, the active turn, and a + * settled summary no fence changes (T3, T15a). */ +function expectStartupReadsTheRow(name: JournalSessionStateCase): void { + const status = readTestJournalSessionStatus(root, name)! + const { state } = loadTestJournal(root, name)! + expect(isUnsettledJournalSessionStatus(status)).toBe(CORPUS_UNSETTLED[name]) + expect(readUnsettledJournalSessionIds(openTestJournalHostDatabase(root).db)).toEqual( + CORPUS_UNSETTLED[name] ? [name] : [] + ) + // The active turn the facts pass finds is the by-sequence reader's. + expect(status.activeTurnId).toBe( + activeStructuredAgentSessionTurnIdBySequence(state.items.values()) + ) + if (CORPUS_UNSETTLED[name]) { + return + } + const snapshot = renderJournalState(state) + for (const fence of [undefined, 0, 3, 4, 100]) { + expect( + projectStructuredAgentSessionStatusState(snapshot.items, snapshot.submissions, fence).summary + ).toEqual(status.summary) + } +} + describe('a failed append leaves the fold equal to a replay of the disk', () => { it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { const journal = await open(name) const violations = failEveryAppendOnce(journal, name) await JOURNAL_SESSION_STATE_CORPUS[name](journal) expect(violations).toEqual([]) + expectStartupReadsTheRow(name) }) it('through every case on one chat, a tombstone and a recreate', async () => { @@ -216,29 +256,32 @@ describe('a failed append leaves the fold equal to a replay of the disk', () => expect(violations).toEqual([]) }) - it.each(JOURNAL_SESSION_STATE_CASES)( - 'with a bookkeeping savepoint that always fails: %s', - async (name) => { - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - vi.spyOn(JournalQueuedMessages.prototype, 'onRowInTransaction').mockImplementation((db) => { - db.prepare( - `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, - queued_sends, live_child_work, summary_json, last_activity_at, rules_version) - VALUES ('savepoint-write', 'running', NULL, 0, 0, 0, '{}', 0, 1)` - ).run() - throw new Error('bookkeeping failed') - }) - const journal = await open(name) - const violations = failEveryAppendOnce(journal, name) - await JOURNAL_SESSION_STATE_CORPUS[name](journal) - expect(violations).toEqual([]) - - const savepointWrite = openTestJournalHostDatabase(root) - .db.prepare("SELECT 1 FROM journal_session_state WHERE session_id = 'savepoint-write'") - .get() - expect(savepointWrite).toBeUndefined() + it('through every case on one chat, with a bookkeeping savepoint that writes and then fails', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let written = 0 + vi.spyOn(JournalQueuedMessages.prototype, 'onRowInTransaction').mockImplementation((db) => { + // A complete row, so only the savepoint's rollback keeps it off the disk. + db.prepare( + `INSERT INTO journal_session_state (session_id, lifecycle, active_turn_id, handed_over_sends, + queued_sends, live_child_work, summary_json, last_activity_at, rules_version, epoch, tip_seq, + tip_ts) VALUES ('savepoint-write', 'running', NULL, 0, 0, 0, '{}', 0, 1, 'epoch-x', 1, 1)` + ).run() + written += 1 + throw new Error('bookkeeping failed') + }) + const journal = await open('mixed') + const violations = failEveryAppendOnce(journal, 'mixed') + for (const name of JOURNAL_SESSION_STATE_CASES) { + await JOURNAL_SESSION_STATE_CORPUS[name](journal).catch(() => undefined) } - ) + expect(violations).toEqual([]) + + expect(written).toBeGreaterThan(0) + const savepointWrite = openTestJournalHostDatabase(root) + .db.prepare("SELECT 1 FROM journal_session_state WHERE session_id = 'savepoint-write'") + .get() + expect(savepointWrite).toBeUndefined() + }) }) describe('a failed append that evicted the oldest settlement id (R3W-1)', () => { diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts index 3b33923f2732..f7f260024390 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-corpus.ts @@ -17,6 +17,7 @@ import { runningTool, send, settledTurn, + startFailure, userMessage } from './journal-session-state-test-writes' @@ -228,6 +229,13 @@ export const JOURNAL_SESSION_STATE_CORPUS = { }) await journal.appendStopEvent({ reason: 'user-stop', turnId: 'turn-2' }, CORPUS_FENCE) }, + // A failed start rejects every send still queued behind it, in the batch that records it. + 'failed start rejects queued': async (journal: AgentSessionJournal) => { + await settledTurn(journal, 'turn-1') + await send(journal, 'send-first', 'first', true) + await send(journal, 'send-second', 'second', true) + await startFailure(journal) + }, // An older build's repair left its notice newest; nothing here owes a rebuild, so the roster counts. 'working roster under a former repair notice': async (journal: AgentSessionJournal) => { await settledTurn(journal, 'turn-1') @@ -266,6 +274,7 @@ export const CORPUS_UNSETTLED: Record = { 'running work settled by a batch': false, 'unknown send recovered': false, "running turn a person's Stop found": true, + 'failed start rejects queued': false, 'working roster under a former repair notice': true } diff --git a/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts index e0980135340a..2df5a33600ea 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state-test-writes.ts @@ -6,6 +6,8 @@ import { type AgentJournalItemIdentity, type AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' +import { agentSessionFailureFact } from '../../../shared/agent-session-failure' +import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' import type { AgentSessionJournal } from './journal-store' /** The fence every case writes its content under; an `unverifiable` turn's writer. */ @@ -94,3 +96,23 @@ export function roster(state: 'working' | 'completed'): AgentJournalItemBody { ] } } + +/** A failed agent start: its notice, and every send still queued rejected, in one batch. */ +export async function startFailure(journal: AgentSessionJournal): Promise { + await journal.appendLifecycleBatch({ + settlementId: 'start-failure:gen-1', + fence: CORPUS_FENCE, + recovered: true, + mutations: [ + { + kind: 'item', + identity: { provider: 'orca', clientMessageId: 'start-failure:gen-1' }, + body: { kind: 'status', tone: 'error', text: 'Codex did not start.' }, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + } + ], + rejectsQueued: agentSessionFailureWords(agentSessionFailureFact('providerStartFailed'), { + surface: 'rejection' + }) + }) +} diff --git a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts index 2e03dc71534c..afd2d0ad89d0 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-state.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-state.test.ts @@ -10,10 +10,6 @@ import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION, type AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' -import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' -import { activeStructuredAgentSessionTurnIdBySequence } from '../../../shared/structured-agent-session-live-turn' -import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' -import { journalPragmaNumber } from './journal-database' import { journalDatabasePath } from './journal-host-database' import Database from '../../sqlite/sync-database' import { @@ -29,14 +25,11 @@ import * as JournalOpen from './journal-open' import { renderJournalState } from './journal-reducer' import { deriveJournalSessionStatus, - isUnsettledJournalSessionStatus, readUnsettledJournalSessionIds, type JournalSessionStatus } from './journal-session-state' import { CORPUS_FENCE, - CORPUS_UNSETTLED, - JOURNAL_SESSION_STATE_CASES, JOURNAL_SESSION_STATE_CORPUS, type JournalSessionStateCase } from './journal-session-state-test-corpus' @@ -143,33 +136,6 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }) }) -describe('the stored status follows every write (T3)', () => { - it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { - const journal = await open(name) - const mismatches: string[] = [] - // Told after each commit, which carried the row and its status together. - journal.observeCommits(() => { - const expected = freshDerivation(name) - try { - expect(stored(name)).toEqual(expected) - } catch { - mismatches.push(`seq ${journal.cursor().sequence}`) - } - }) - await JOURNAL_SESSION_STATE_CORPUS[name](journal) - expect(mismatches).toEqual([]) - expect(stored(name)).toEqual(freshDerivation(name)) - expect(isUnsettledJournalSessionStatus(stored(name)!)).toBe(CORPUS_UNSETTLED[name]) - expect(readUnsettledJournalSessionIds(db()).includes(name)).toBe(CORPUS_UNSETTLED[name]) - // The active turn the facts pass finds is the by-sequence reader's. - expect(stored(name)?.activeTurnId).toBe( - activeStructuredAgentSessionTurnIdBySequence( - loadTestJournal(root, name)!.state.items.values() - ) - ) - }) -}) - describe('observers hear of a write only once it is committed (T13)', () => { it('tells observers after COMMIT, when another connection already reads the row and its status', async () => { const journal = await open('committed') @@ -380,27 +346,4 @@ describe('epoch writes carry the status (T10)', () => { expect(journal.epoch).toBe(epoch) expect(loadTestJournal(root, 'settled')?.state.epoch).toBe(epoch) }) - - it('leaves the schema at version 4, so an older build stays writable', async () => { - await write('settled') - expect(JOURNAL_DB_SCHEMA_VERSION).toBe(4) - expect(journalPragmaNumber(db(), 'user_version')).toBe(4) - }) -}) - -describe('a stored summary is fence-independent once settled (T15a, regression guard)', () => { - it.each(JOURNAL_SESSION_STATE_CASES)('%s', async (name) => { - await write(name) - const derived = freshDerivation(name) - if (isUnsettledJournalSessionStatus(derived)) { - return - } - const snapshot = renderJournalState(loadTestJournal(root, name)!.state) - for (const fence of [undefined, 0, 3, 4, 100]) { - expect( - projectStructuredAgentSessionStatusState(snapshot.items, snapshot.submissions, fence) - .summary - ).toEqual(derived.summary) - } - }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts index c5df1d22c975..e8a4df7fb1f7 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-rules.test.ts @@ -19,10 +19,11 @@ import { JOURNAL_SESSION_STATE_CORPUS } from './journal-session-state-test-corpus' -// Recorded with the rules version: change both together, after checking the new rows are right. +// Recorded with the rules version: change both together, after checking the new rows are right. A +// new corpus case changes only the digest: record it, with no bump. const DERIVED = { rules: 3, - digest: '7ea161678a4408a2c3b0648f6c781763cab230a6ffc1d80a4aed9c1ad03a0bc6' + digest: 'f0118b4048c1f6e1feb53d2105557019564c08d4600c69f200a2abbd44ba1d15' } const journals = createTrackedJournalOpener() From 85858a8f418df69fdf447d275d5ed01554f7b7b0 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:42:28 -0700 Subject: [PATCH 104/107] test(native-chat): fold overlapping settlement and startup-gate tests into the cases that already cover them The queued-leftover settle check runs inside the corpus-by-evidence agreement test (every rejected leftover is checked, and the no-record plan for the queued case); the "never selects an unverifiable turn" test is dropped, since that agreement test pins it for every evidence. The two host-build failures that open the startup gate run as one table; the slow-build-past-the-ceiling pair is dropped, as the gate-level ceiling table covers each outcome. --- ...ured-agent-session-open-settlement.test.ts | 31 ++------- ...-agent-session-startup-restoration.test.ts | 64 ++++++++----------- 2 files changed, 33 insertions(+), 62 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts index eac94d9935af..f3c080209aa6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts @@ -109,6 +109,10 @@ describe('the stored status and the plan agree (T4)', () => { const revisesVerdictOnly = name === 'unverifiable turn' && deathEvidence?.ownerFence === CORPUS_FENCE expect(selected(sessionId)).toBe(!openSettlementPlanIsEmpty(plan) && !revisesVerdictOnly) + if (name === 'queued leftover') { + // A queued leftover is owed with no record to read, too. + expect(planOpenSettlement(journal, null).leftoverQueued).toEqual(['send-queued']) + } // The open appends exactly the plan: a row per roster, per recovered send and per rejected // leftover, and one lifecycle batch for what the gone generation left. @@ -126,6 +130,9 @@ describe('the stored status and the plan agree (T4)', () => { ) // Every entry revised its entity out of the state that selected it (T4b, T15b). + for (const clientMessageId of plan.leftoverQueued) { + expect(journal.submission(clientMessageId)).toMatchObject({ dispatchState: 'rejected' }) + } expect(selected(sessionId)).toBe(false) expect(openSettlementPlanIsEmpty(planOpenSettlement(journal, record))).toBe(true) }) @@ -223,28 +230,4 @@ describe('the stored status and the plan agree (T4)', () => { expect(storedStatus('unkeyed')).toMatchObject({ lifecycle: 'idle' }) expect(selected('unkeyed')).toBe(false) }) - - it('never selects a chat to revise a verdict: an unverifiable turn stays as it settled (D16 gone)', async () => { - const journal = await open('unverifiable') - await JOURNAL_SESSION_STATE_CORPUS['unverifiable turn'](journal) - expect(storedStatus('unverifiable')).toMatchObject({ lifecycle: 'idle' }) - expect(selected('unverifiable')).toBe(false) - }) - - it('selects a chat whose only debt is a queued leftover, and settles it (T15b)', async () => { - const writer = await open('queued') - await JOURNAL_SESSION_STATE_CORPUS['queued leftover'](writer) - await writer.close() - const journal = await open('queued') - expect(storedStatus('queued')).toMatchObject({ lifecycle: 'idle', queuedSends: 1 }) - const plan = planOpenSettlement(journal, null) - expect(plan.leftoverQueued).toEqual(['send-queued']) - - await appendOpenSettlement(journal, plan, CORPUS_FENCE, (error) => { - throw error - }) - - expect(journal.submission('send-queued')).toMatchObject({ dispatchState: 'rejected' }) - expect(storedStatus('queued')).toMatchObject({ queuedSends: 0, summary: { status: 'idle' } }) - }) }) diff --git a/src/main/runtime/structured-agent-session-startup-restoration.test.ts b/src/main/runtime/structured-agent-session-startup-restoration.test.ts index 8af2aaadcab7..c289eb651662 100644 --- a/src/main/runtime/structured-agent-session-startup-restoration.test.ts +++ b/src/main/runtime/structured-agent-session-startup-restoration.test.ts @@ -94,26 +94,34 @@ it('holds chat commands until the startup settle ends, and then lets them throug ) }) -it('opens the gate when the startup step fails, never stranding a command', async () => { - await rig.crash() - await rig.boot() - const runtime = restartedRuntime() - const gate = internals(runtime).structuredAgentSessionStartupGate - internals(runtime).ensureStructuredAgentSessionHost = async () => { - throw new Error('host refused') - } - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - runtime.holdStructuredAgentSessionCommandsForStartup() +it.each([ + ['throws', 'step failed'], + ['builds no host', 'no host'] +] as const)( + 'opens the gate when the host build %s, never stranding a command', + async (how, outcome) => { + await rig.crash() + await rig.boot() + const runtime = restartedRuntime() + const gate = internals(runtime).structuredAgentSessionStartupGate + internals(runtime).ensureStructuredAgentSessionHost = async () => { + if (how === 'throws') { + throw new Error('host refused') + } + } + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + runtime.holdStructuredAgentSessionCommandsForStartup() - const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) - await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) + const timing = vi.spyOn(console, 'info').mockImplementation(() => undefined) + await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) - expect(gate.ready()).toBeNull() - expect(timing).toHaveBeenCalledOnce() - expect(timing).toHaveBeenCalledWith( - expect.stringMatching(/\(step failed\); opened \+\d+ ms by step failed$/) - ) -}) + expect(gate.ready()).toBeNull() + expect(timing).toHaveBeenCalledOnce() + expect(timing).toHaveBeenCalledWith( + expect.stringMatching(new RegExp(`\\(${outcome}\\); opened \\+\\d+ ms by ${outcome}$`)) + ) + } +) it('lets held commands go before the shortest client timeout on a held call (the AI vault restore, 5 s)', () => { vi.spyOn(console, 'warn').mockImplementation(() => undefined) @@ -212,26 +220,6 @@ describe('one timing line per launch, with the real step times whoever opened th await expectOneCeilingLine('settle ended', /\+([5-9]|\d{2,}) ms/) }) - - it.each([ - ['throws', 'step failed'], - ['builds no host', 'no host'] - ] as const)('writes it when a slow host build past the ceiling %s', async (how, outcome) => { - const runtime = restartedRuntime() - const internal = internals(runtime) - internal.structuredAgentSessionStartupGate = new StructuredAgentSessionStartupGate(CEILING_MS) - internal.ensureStructuredAgentSessionHost = async () => { - await sleep(30) - if (how === 'throws') { - throw new Error('host build failed') - } - } - runtime.holdStructuredAgentSessionCommandsForStartup() - - await runtime.prepareStructuredAgentSessionStartupRestoration().catch(() => undefined) - - await expectOneCeilingLine(outcome) - }) }) it('seeds and settles on a host no client ever lists (T8)', async () => { From 6170407f5856a94102f5c1f1d626f52a7205a407 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:44:36 -0700 Subject: [PATCH 105/107] refactor(native-chat): the startup seed drops its closed-tab check, which nothing can reach now The seed checked each listed chat still had its tab because the catch-up and the per-chat-file restore awaited between the listing and the seed. With both gone, the listing's ids and the seed run in the same task, so no tab can close between them. --- .../agent-session-wire/structured-agent-session-reveal.ts | 1 - .../structured-agent-session-startup-state.ts | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 56a49f452d19..051db78066a9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -111,7 +111,6 @@ export function createStructuredAgentSessionHostRestore( recoveryBudgetMs: deps.startupRecoveryBudgetMs, serialize: rest.serialize, hasSession: rest.hasSession, - isListed: readRestore.isListed, isDisposed: rest.isDisposed }) // Startup's lease bookkeeping: no lease is checked again, or recovered twice. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts index 7f5e9eee441a..901db056bf24 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.ts @@ -60,8 +60,6 @@ export type StructuredAgentSessionStartupStateDeps = { recoveryBudgetMs?: number serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean - /** Whether the chat still has its tab: one closed during startup is not seeded. */ - isListed: (sessionId: string) => boolean isDisposed: () => boolean } @@ -136,8 +134,8 @@ function seedStoredStatuses( const byId = new Map(stored.map((row) => [row.sessionId, row.status])) const background: string[] = [] for (const sessionId of listedIds) { - // Its tab closed during startup, or its open (a restore, read or send) publishes its own status. - if (!deps.isListed(sessionId) || deps.hasSession(sessionId)) { + // Its open (a restore, read or send) publishes its own status. + if (deps.hasSession(sessionId)) { continue } const record = deps.openDeps.store.getRecord(sessionId) From 559dab177504a2b9bb4993bf1e19706241c0cf49 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:53:22 -0700 Subject: [PATCH 106/107] test(native-chat): the tip check is pinned for a chat moved to a new epoch, and the lease-check table names every row and checks the listed chat too Breaking the tip check's epoch condition left every test green: no case moved a chat to a new epoch whose newest row matched the old tip. One does now. The lease-check permutations ran through it.each, whose interpolated titles are shortened past 40 characters, so a test filter could not select the two rows where a command completes the check. They run as a loop now, and each row also checks the listed chat's verdict, which the restore's own lease answers decide. --- .../journal-session-status-tip.test.ts | 15 +++++++++++++++ ...uctured-agent-session-startup-state.test.ts | 18 ++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts index 118abed8c036..4893a50b79fc 100644 --- a/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-session-status-tip.test.ts @@ -15,6 +15,7 @@ import { liveTestJournalRows, loadTestJournal, openTestJournalHostDatabase, + publishTestJournalEpoch, readTestJournalSessionStatus } from './journal-host-database-test-support' import { @@ -125,6 +126,20 @@ it('reads a row as missing when a repair in the same epoch regrew the history to expect(readUnsettledJournalSessionIds(db())).toEqual([]) }) +it('reads a row as missing when an older build moved the chat to a new epoch, even at the same tip', async () => { + const journal = await open('chat') + await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) + await journal.close() + const tip = liveTestJournalRows(db(), 'chat').at(-1)! + + // A new epoch whose newest row matches the old tip's sequence and time. + publishTestJournalEpoch(db(), 'chat', 'epoch-older-build') + insertTestJournalRowJson(db(), 'chat', tip.seq, tip.rowJson, tip.ts) + + expect(stored('chat')).toBeNull() + expect(readUnsettledJournalSessionIds(db())).toEqual([]) +}) + it('never selects the row of a chat whose history is gone', async () => { const journal = await open('chat') await JOURNAL_SESSION_STATE_CORPUS['running tool'](journal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts index ed92fc7bf4f7..b55c136979c0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts @@ -416,7 +416,7 @@ describe('a provider process that outlived the crash (R1T-2)', () => { expect(stopOwnerProcess).not.toHaveBeenCalled() }) - it.each([ + const LEASE_CHECK_FAILURES = [ { check: "'s first try fails", fails: 1, command: null, checks: 2, stopped: 1 }, { check: ' fails twice', fails: 2, command: null, checks: 2, stopped: 0 }, { @@ -433,9 +433,10 @@ describe('a provider process that outlived the crash (R1T-2)', () => { checks: 3, stopped: 1 } - ] as const)( - 'settles an unlisted chat from what its recovery found when the lease check$check', - async ({ fails, command, checks, stopped }) => { + ] as const + // A loop, not it.each: it.each shortens a long interpolated title, which `-t` then cannot match. + for (const { check, fails, command, checks, stopped } of LEASE_CHECK_FAILURES) { + it(`settles each chat from what its recovery found when the lease check${check}`, async () => { const rig = await newRig() const stopOwnerProcess = await crashWithLiveOwner(rig, ['session-listed', 'session-closed']) const listed = listedIds(rig) @@ -478,8 +479,13 @@ describe('a provider process that outlived the crash (R1T-2)', () => { stopped ? { deathEvidence: { kind: 'pid-absent' } } : { unreconciled: true } ) expect(rig.host.hasSession('session-closed')).toBe(false) - } - ) + // The listed chat is settled by the listed worker from the same lease answers. + expect(readTestJournalSessionStatus(rig.root, 'session-listed')).toMatchObject({ + lifecycle: 'idle', + summary: { turnOutcome: stopped ? 'interruption' : 'unconfirmed' } + }) + }) + } }) /** Chats whose turn was running when Orca died; the first is listed. The last one's provider From dce10f42b8db73e05630063862d7b1a10f2b7664 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:44:30 -0700 Subject: [PATCH 107/107] test(native-chat): run this PR's real-database startup tests in the Node runtime project --- config/scripts/vitest-sqlite-runtime-files.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index c8705ab6cc7a..ff15239f6334 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -83,8 +83,10 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-main-agent-working-agreement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-newer-content-host.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-open-settlement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-persisted-tab-listing.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-pre-spawn-first-answer.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.test.ts', @@ -106,10 +108,12 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restore-after-listing.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restore-without-import.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-send-during-startup-lease-check.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts', @@ -117,7 +121,9 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-damaged-chat.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-state.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-child-work.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-child.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-children.test.ts', @@ -182,6 +188,8 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/runtime/structured-agent-session-rollback-compatibility.test.ts', 'src/main/runtime/structured-agent-session-spawn-token-descendants.test.ts', 'src/main/runtime/structured-agent-session-startup-reconcile.test.ts', + 'src/main/runtime/structured-agent-session-startup-restoration.test.ts', + 'src/main/runtime/structured-agent-session-tab-listing-startup.test.ts', 'src/main/runtime/structured-claude-pending-rewind.test.ts', 'src/main/ssh-expired-lease-pane-readoption.test.ts', 'src/main/ssh-reattach-pane-cardinality.test.ts',